Skip to content
This repository was archived by the owner on Nov 24, 2025. It is now read-only.

Commit 79a78b7

Browse files
thxCodethxCode
committed
refactor: support reasoning content 2
Signed-off-by: thxCode <thxcode0824@gmail.com>
1 parent 800d59c commit 79a78b7

1 file changed

Lines changed: 65 additions & 50 deletions

File tree

‎llama-box/httpserver.hpp‎

Lines changed: 65 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -3537,12 +3537,10 @@ struct httpserver {
35373537
inline int32_t decode_completion_task_batch(llama_context * input_ctx, llama_batch & input_batch,
35383538
const std::vector<std::unique_ptr<btask>> & batch_task_ptrs) {
35393539
// decoded results:
3540-
// -3 compute failed,
3541-
// -2 allocate failed,
3542-
// -1 no tokens,
3540+
// -2 no update / failed to compute / failed ot reserve output buffer,
3541+
// -1 no tokens / failed to allocate batch,
35433542
// 0 ok,
3544-
// 1 no kv cache,
3545-
// 2 compute aborted
3543+
// 1 failed to prepare.
35463544
int32_t decoded = 0;
35473545
bool is_mrope_view = llm_model_rope_mrope && input_batch.n_tokens > batch_view_max;
35483546
std::vector<llama_pos> mrope_pos_view;
@@ -3603,9 +3601,7 @@ struct httpserver {
36033601
}
36043602
// shift the target task if found
36053603
if (task != nullptr) {
3606-
int32_t pos_previous = task->pos;
36073604
shift_completion_task_cache(task);
3608-
task->pos -= (pos_previous - task->pos);
36093605
// adjust batch pos
36103606
input_batch.pos[task->i_batch_seq_end] = task->pos - 1;
36113607
}
@@ -3748,8 +3744,8 @@ struct httpserver {
37483744
SRV_INF(
37493745
"rid %s | "
37503746
"batching, waiting previous batch finished: not enough space to place all tokens, "
3751-
"llm_kv_cache_used(%d) - llm_kv_cache_inactive(%d) + n_prefilling_request(%d) > "
3752-
"llm_kv_cache_limit(%d)\n",
3747+
"kv_cache_used(%d) - kv_cache_inactive(%d) + prefill_t(%d) > "
3748+
"kv_cache_limit(%d)\n",
37533749
rid.c_str(), llm_kv_cache_used, llm_kv_cache_inactive, task->n_prefilling_request,
37543750
llm_kv_cache_limit);
37553751
process_tasks->enqueue(std::move(task_ptr));
@@ -3996,18 +3992,6 @@ struct httpserver {
39963992
llm_kv_cache_used += n_text_d;
39973993
// abort if incomplete prefilling
39983994
if (task->n_prefilled < task->n_prefilling_request) {
3999-
// clean prompt cache
4000-
if (cache_prompt) {
4001-
cache_prompt_entry & cache = cache_prompts.at(seq_id);
4002-
cache.tokens.clear();
4003-
cache.used = false;
4004-
cache.pos = 0;
4005-
cache.pos_discard = 0;
4006-
SRV_INFV(2,
4007-
"rid %s | released cache prompt, "
4008-
"seq = %d\n",
4009-
rid.c_str(), seq_id);
4010-
}
40113995
// clean kv cache
40123996
llama_memory_seq_rm(llama_get_memory(llm_ctx), seq_id, 0, -1);
40133997
if (llm_ctx_draft != nullptr) {
@@ -4018,6 +4002,18 @@ struct httpserver {
40184002
"clean kv cache, seq %d = [0, end)\n",
40194003
rid.c_str(), seq_id);
40204004
llm_kv_cache_used -= task->pos;
4005+
// clean prompt cache
4006+
if (cache_prompt) {
4007+
cache_prompt_entry & cache = cache_prompts.at(seq_id);
4008+
cache.tokens.clear();
4009+
cache.used = false;
4010+
cache.pos = 0;
4011+
cache.pos_discard = 0;
4012+
SRV_INFV(2,
4013+
"rid %s | released cache prompt, "
4014+
"seq = %d, kv_cache_used = %d, kv_cache_inactive = %d\n",
4015+
rid.c_str(), seq_id, llm_kv_cache_used, llm_kv_cache_inactive);
4016+
}
40214017
// output
40224018
json data = {
40234019
{ "message",
@@ -4201,18 +4197,6 @@ struct httpserver {
42014197
auto * task = dynamic_cast<completions_task *>(task_ptr.get());
42024198
const std::string rid = task->get_r_id();
42034199
const int32_t seq_id = task->get_seq_id();
4204-
// clean prompt cache
4205-
if (cache_prompt) {
4206-
cache_prompt_entry & cache = cache_prompts.at(seq_id);
4207-
cache.tokens.clear();
4208-
cache.used = false;
4209-
cache.pos = 0;
4210-
cache.pos_discard = 0;
4211-
SRV_INFV(2,
4212-
"rid %s | released cache prompt, "
4213-
"seq = %d\n",
4214-
rid.c_str(), seq_id);
4215-
}
42164200
// clean kv cache
42174201
llama_memory_seq_rm(llama_get_memory(llm_ctx), seq_id, 0, -1);
42184202
if (llm_ctx_draft != nullptr) {
@@ -4223,6 +4207,18 @@ struct httpserver {
42234207
"clean kv cache, seq %d = [0, end)\n",
42244208
rid.c_str(), seq_id);
42254209
llm_kv_cache_used -= task->pos;
4210+
// clean prompt cache
4211+
if (cache_prompt) {
4212+
cache_prompt_entry & cache = cache_prompts.at(seq_id);
4213+
cache.tokens.clear();
4214+
cache.used = false;
4215+
cache.pos = 0;
4216+
cache.pos_discard = 0;
4217+
SRV_INFV(2,
4218+
"rid %s | released cache prompt, "
4219+
"seq = %d, kv_cache_used = %d, kv_cache_inactive = %d\n",
4220+
rid.c_str(), seq_id, llm_kv_cache_used, llm_kv_cache_inactive);
4221+
}
42264222
// output
42274223
json data = {
42284224
{ "message",
@@ -4260,8 +4256,8 @@ struct httpserver {
42604256
cache.pos_discard = 0;
42614257
SRV_INFV(2,
42624258
"rid %s | released cache prompt, "
4263-
"seq = %d\n",
4264-
rid.c_str(), seq_id);
4259+
"seq = %d, kv_cache_used = %d, kv_cache_inactive = %d\n",
4260+
rid.c_str(), seq_id, llm_kv_cache_used, llm_kv_cache_inactive);
42654261
}
42664262
// clean kv cache
42674263
llama_memory_seq_rm(llama_get_memory(llm_ctx), seq_id, 0, -1);
@@ -4362,20 +4358,31 @@ struct httpserver {
43624358
if (support_reasoning && !task->reasoning_finished) {
43634359
// find reasoning begin [in token]
43644360
if (!task->reasoning_start_found) {
4365-
task->reasoning_start_found = tok == reasoning_start_token;
4366-
if (!reasoning_in_content && task->reasoning_start_found) {
4367-
continue;
4361+
if (reasoning_start_token != LLAMA_TOKEN_NULL) {
4362+
task->reasoning_start_found = tok == reasoning_start_token;
4363+
if (task->reasoning_start_found) {
4364+
// ignore reasoning start content if needed
4365+
if (!reasoning_in_content) {
4366+
continue;
4367+
}
4368+
}
4369+
// finish reasoning analysis as not found any available start
4370+
else {
4371+
task->reasoning_finished = true;
4372+
}
43684373
}
43694374
}
43704375
// find reasoning end [in token]
43714376
else if (!task->reasoning_end_found) {
4372-
if (tok == reasoning_end_token) {
4373-
task->reasoning_end_found = true;
4374-
if (!reasoning_in_content) {
4375-
continue;
4376-
}
4377-
} else {
4377+
if (reasoning_end_token != LLAMA_TOKEN_NULL) {
43784378
task->n_reasoning++;
4379+
task->reasoning_end_found = tok == reasoning_end_token;
4380+
if (task->reasoning_end_found) {
4381+
// ignore reasoning end content if needed
4382+
if (!reasoning_in_content) {
4383+
continue;
4384+
}
4385+
}
43794386
}
43804387
}
43814388
// finish
@@ -4410,27 +4417,35 @@ struct httpserver {
44104417
break;
44114418
}
44124419
}
4413-
// find reasoning
4414-
if (support_reasoning && !task->reasoning_finished) {
4420+
// has reasoning
4421+
if (support_reasoning && !task->reasoning_finished &&
4422+
reasoning_start_token == LLAMA_TOKEN_NULL) {
44154423
// find reasoning begin [in word]
4416-
if (!task->reasoning_start_found && reasoning_start_token == LLAMA_TOKEN_NULL) {
4424+
if (!task->reasoning_start_found) {
44174425
send_text = false; // avoid to send text before reasoning start
44184426
task->reasoning_start_found =
44194427
string_starts_with(task->generated_text, reasoning_start_word);
44204428
if (task->reasoning_start_found) {
44214429
send_text = true;
44224430
task->n_reasoning++;
4431+
// ignore reasoning start content if needed
44234432
if (!reasoning_in_content) {
44244433
task->generated_text.clear();
44254434
}
44264435
}
4436+
// finish reasoning analysis as not found any available start
4437+
else if (task->generated_text.length() > reasoning_start_word.length()) {
4438+
task->reasoning_finished = true;
4439+
send_text = true;
4440+
}
44274441
}
44284442
// find reasoning end [in word]
4429-
else if (!task->reasoning_end_found && reasoning_end_token == LLAMA_TOKEN_NULL) {
4443+
else if (!task->reasoning_end_found) {
44304444
task->n_reasoning++;
44314445
task->reasoning_end_found =
44324446
string_ends_with(task->generated_text, reasoning_end_word);
44334447
if (task->reasoning_end_found) {
4448+
// ignore reasoning end content if needed
44344449
if (!reasoning_in_content) {
44354450
size_t pos = task->generated_text.rfind(reasoning_end_word);
44364451
task->generated_text_keep_pos = pos;
@@ -4802,8 +4817,8 @@ struct httpserver {
48024817
llm_kv_cache_inactive += task->pos;
48034818
SRV_INFV(2,
48044819
"rid %s | released cache prompt, "
4805-
"seq = %d\n",
4806-
rid.c_str(), seq_id);
4820+
"seq = %d, kv_cache_used = %d, kv_cache_inactive = %d\n",
4821+
rid.c_str(), seq_id, llm_kv_cache_used, llm_kv_cache_inactive);
48074822
}
48084823
}
48094824
return;

0 commit comments

Comments
 (0)