Skip to content
This repository was archived by the owner on Nov 24, 2025. It is now read-only.

Commit 437e804

Browse files
thxCodethxCode
committed
refactor: adjust context size
Signed-off-by: thxCode <thxcode0824@gmail.com>
1 parent 79a78b7 commit 437e804

1 file changed

Lines changed: 27 additions & 25 deletions

File tree

‎llama-box/httpserver.hpp‎

Lines changed: 27 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -2807,8 +2807,9 @@ struct httpserver {
28072807
}
28082808
llm_vocab = llama_model_get_vocab(llm_model);
28092809
llm_ctx_size = int32_t(llama_n_ctx(llm_ctx));
2810+
llm_slot_ctx_size = llm_ctx_size / llm_params.n_parallel;
28102811
llm_ctx_embed_size = llama_model_n_embd(llm_model);
2811-
llm_kv_cache_limit = llm_ctx_size - 1;
2812+
llm_kv_cache_limit = llm_slot_ctx_size - 1;
28122813
llm_kv_cache_shift = llama_memory_can_shift(llama_get_memory(llm_ctx));
28132814
// NB(thxCode): llama_causal_attn is a patch.
28142815
llm_model_casual = llama_causal_attn(llm_ctx);
@@ -3130,7 +3131,7 @@ struct httpserver {
31303131
// sample tokens per second
31313132
if (params.n_tps < 0) {
31323133
SRV_INF("%s", "sampling tokens per second, this will take some time...\n");
3133-
const int32_t n_check = std::min(llm_ctx_size, params.llm_params.n_ubatch);
3134+
const int32_t n_check = std::min(llm_slot_ctx_size, params.llm_params.n_ubatch);
31343135
llama_tokens check_prompt_tokens = { llama_vocab_bos(llm_vocab) };
31353136
common_sampler * check_smpl = common_sampler_init(llm_model, params.llm_params.sampling);
31363137
int64_t t_start_decode = ggml_time_us();
@@ -3352,6 +3353,7 @@ struct httpserver {
33523353
llama_context * llm_ctx = nullptr;
33533354
const llama_vocab * llm_vocab = nullptr;
33543355
int32_t llm_ctx_size = 0;
3356+
int32_t llm_slot_ctx_size = 0;
33553357
int32_t llm_ctx_embed_size = 0;
33563358
int32_t llm_kv_cache_used = 0; // include llm_kv_cache_inactive
33573359
int32_t llm_kv_cache_inactive = 0;
@@ -3705,14 +3707,14 @@ struct httpserver {
37053707
}
37063708
}
37073709
} else if (batch_task_type != ttype) {
3708-
SRV_INF(
3710+
SRV_DBG(
37093711
"rid %s | "
37103712
"batching, waiting previous batch finished: not the same kind batch\n",
37113713
rid.c_str());
37123714
process_tasks->enqueue(std::move(task_ptr));
37133715
continue;
37143716
} else if (!equal_lora(task_ptr->get_lora_adapters(), lora_adapters)) {
3715-
SRV_INF(
3717+
SRV_DBG(
37163718
"rid %s | "
37173719
"batching, waiting previous batch finished: lora adapters not matched\n",
37183720
rid.c_str());
@@ -3741,7 +3743,7 @@ struct httpserver {
37413743
// filter
37423744
if (llm_kv_cache_used - llm_kv_cache_inactive + task->n_prefilling_request >
37433745
llm_kv_cache_limit) {
3744-
SRV_INF(
3746+
SRV_DBG(
37453747
"rid %s | "
37463748
"batching, waiting previous batch finished: not enough space to place all tokens, "
37473749
"kv_cache_used(%d) - kv_cache_inactive(%d) + prefill_t(%d) > "
@@ -4097,8 +4099,8 @@ struct httpserver {
40974099
if (task->i_input_prefilled < n_input) {
40984100
llama_tokens tokenized_input = task->tokenized_inputs[task->i_input_prefilled];
40994101
const auto n_pos = int32_t(tokenized_input.size());
4100-
// allow batch's tokens size be equal to llm_ctx_size
4101-
if (batch_text.n_tokens + n_pos > llm_ctx_size) {
4102+
// allow batch's tokens size be equal to llm_slot_ctx_size
4103+
if (batch_text.n_tokens + n_pos > llm_slot_ctx_size) {
41024104
SRV_INF("rid %s | batching, not enough space to fill, waiting\n", rid.c_str());
41034105
continue;
41044106
}
@@ -5500,8 +5502,8 @@ struct httpserver {
55005502
{ "n_params", llama_model_n_params(llm_model) },
55015503
{ "size", llama_model_size(llm_model) },
55025504
{ "n_ctx", llm_ctx_size },
5503-
{ "n_slot", 1 },
5504-
{ "n_slot_ctx", llm_ctx_size },
5505+
{ "n_slot", params.llm_params.n_threads_http },
5506+
{ "n_slot_ctx", llm_slot_ctx_size },
55055507
{ "ctx_shift", shift_context },
55065508
{ "prompt_cache", cache_prompt },
55075509
{ "seed", int32_t(params.llm_params.sampling.seed) },
@@ -5589,16 +5591,16 @@ struct httpserver {
55895591
{
55905592
llama_tokens tokenized_prompt = tokenize_prompt(llm_vocab, req->prompt, true, true);
55915593
n_prefilling_request = int32_t(tokenized_prompt.size());
5592-
if (n_prefilling_request >= llm_ctx_size) {
5594+
if (n_prefilling_request >= llm_slot_ctx_size) {
55935595
if (!shift_context) {
55945596
SRV_ERR(
55955597
"rid %s | prompt tokens size exceeds the context size, please enable context shift "
55965598
"or reduce prompt size, prefill_t = %d, n_ctx = %d\n",
5597-
req->get_id(), n_prefilling_request, llm_ctx_size);
5599+
req->get_id(), n_prefilling_request, llm_slot_ctx_size);
55985600
return send_json(request, response, httplib::BadRequest_400,
55995601
"Illegal param: prompt tokens size exceeds the context size");
56005602
}
5601-
const int32_t n_left = llm_ctx_size - params.llm_params.n_keep;
5603+
const int32_t n_left = llm_slot_ctx_size - params.llm_params.n_keep;
56025604
const int32_t n_block_size = n_left >> 1;
56035605
const int32_t n_block_erased =
56045606
(n_prefilling_request - params.llm_params.n_keep - n_block_size) / n_block_size;
@@ -5621,7 +5623,7 @@ struct httpserver {
56215623
return send_json(request, response, httplib::BadRequest_400, "Illegal param: empty completions tokens");
56225624
}
56235625

5624-
int32_t n_decoding_budget = llm_ctx_size;
5626+
int32_t n_decoding_budget = llm_slot_ctx_size;
56255627
if (req->max_tokens > 0) {
56265628
n_decoding_budget = req->max_tokens;
56275629
} else if (req->max_tokens < 0) {
@@ -5709,16 +5711,16 @@ struct httpserver {
57095711
if (req->multimedias.empty()) {
57105712
llama_tokens tokenized_prompt = tokenize_prompt(llm_vocab, req->chat_params.prompt, true, true);
57115713
n_prefilling_request = int32_t(tokenized_prompt.size());
5712-
if (n_prefilling_request >= llm_ctx_size) {
5714+
if (n_prefilling_request >= llm_slot_ctx_size) {
57135715
if (!shift_context) {
57145716
SRV_ERR(
57155717
"rid %s | prompt tokens size exceeds the context size, please enable context shift "
57165718
"or reduce prompt size, prefill_t = %d, n_ctx = %d\n",
5717-
req->get_id(), n_prefilling_request, llm_ctx_size);
5719+
req->get_id(), n_prefilling_request, llm_slot_ctx_size);
57185720
return send_json(request, response, httplib::BadRequest_400,
57195721
"Illegal param: prompt tokens size exceeds the context size");
57205722
}
5721-
const int32_t n_left = llm_ctx_size - params.llm_params.n_keep;
5723+
const int32_t n_left = llm_slot_ctx_size - params.llm_params.n_keep;
57225724
const int32_t n_block_size = n_left >> 1;
57235725
const int32_t n_block_erased =
57245726
(n_prefilling_request - params.llm_params.n_keep - n_block_size) / n_block_size;
@@ -6051,11 +6053,11 @@ struct httpserver {
60516053
tokenized_prompts.emplace_back(std::move(tokenized_text));
60526054
}
60536055

6054-
if (n_prefilling_request >= llm_ctx_size) {
6056+
if (n_prefilling_request >= llm_slot_ctx_size) {
60556057
SRV_ERR(
60566058
"rid %s | prompt tokens size exceeds the context size, please increase the context size "
60576059
"or reduce prompt size, prefill_t = %d, n_ctx = %d\n",
6058-
req->get_id(), n_prefilling_request, llm_ctx_size);
6060+
req->get_id(), n_prefilling_request, llm_slot_ctx_size);
60596061
return send_json(request, response, httplib::BadRequest_400,
60606062
"Illegal param: prompt tokens size exceeds the context size");
60616063
}
@@ -6071,7 +6073,7 @@ struct httpserver {
60716073

60726074
bool tokenized_prompts_include_tools = !req->tools.empty();
60736075

6074-
int32_t n_decoding_budget = llm_ctx_size;
6076+
int32_t n_decoding_budget = llm_slot_ctx_size;
60756077
if (req->max_tokens > 0) {
60766078
n_decoding_budget = req->max_tokens;
60776079
} else if (req->max_tokens < 0) {
@@ -6149,17 +6151,17 @@ struct httpserver {
61496151
tokenized_inputs[i].push_back(tok_eos);
61506152
}
61516153
auto n_pos = int32_t(tokenized_inputs[i].size());
6152-
if (n_pos > llm_ctx_size) {
6154+
if (n_pos > llm_slot_ctx_size) {
61536155
if (!shift_context) {
61546156
return send_json(request, response, httplib::BadRequest_400,
61556157
"Illegal param: \"input\" tokens size exceeds the context size");
61566158
}
61576159
SRV_WRN(
61586160
"rid %s | input item %zu tokens size exceeds the context size, "
61596161
"shifting context [%d, %d) -> [0, %d)\n",
6160-
req->get_id(), i, n_pos - llm_ctx_size, n_pos, llm_ctx_size);
6161-
tokenized_inputs[i].erase(tokenized_inputs[i].begin(), tokenized_inputs[i].end() - llm_ctx_size);
6162-
n_pos = llm_ctx_size;
6162+
req->get_id(), i, n_pos - llm_slot_ctx_size, n_pos, llm_slot_ctx_size);
6163+
tokenized_inputs[i].erase(tokenized_inputs[i].begin(), tokenized_inputs[i].end() - llm_slot_ctx_size);
6164+
n_pos = llm_slot_ctx_size;
61636165
}
61646166
}
61656167

@@ -6190,14 +6192,14 @@ struct httpserver {
61906192
const size_t n_tok_addition = 4;
61916193

61926194
llama_tokens tokenized_query = tokenize_prompt(llm_vocab, req->query, false, true);
6193-
if (req->normalize && tokenized_query.size() * 2 + n_tok_addition > size_t(llm_ctx_size)) {
6195+
if (req->normalize && tokenized_query.size() * 2 + n_tok_addition > size_t(llm_slot_ctx_size)) {
61946196
return send_json(
61956197
request, response, httplib::BadRequest_400,
61966198
R"(Illegal param: "query" length exceeds the context size, disable "normalize" to bypass this check)");
61976199
}
61986200
auto decorate = [&](const llama_tokens & tokenized_document) {
61996201
auto n_pos = int32_t(tokenized_query.size() + tokenized_document.size() + n_tok_addition);
6200-
if (n_pos > llm_ctx_size) {
6202+
if (n_pos > llm_slot_ctx_size) {
62016203
throw std::invalid_argument(
62026204
R"(Illegal param: the sum of the lengths of "query" and "document" exceeds the context size)");
62036205
}

0 commit comments

Comments
 (0)