Skip to content
This repository was archived by the owner on Nov 24, 2025. It is now read-only.

Commit 42aa233

Browse files
thxCodethxCode
committed
refactor: adjust clip/draft loading sequence
Signed-off-by: thxCode <thxcode0824@gmail.com>
1 parent 52d2def commit 42aa233

1 file changed

Lines changed: 52 additions & 48 deletions

File tree

‎llama-box/httpserver.hpp‎

Lines changed: 52 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -2732,6 +2732,54 @@ struct httpserver {
27322732
}
27332733
}
27342734

2735+
// load multimodal projection model if needed.
2736+
if (!params.llm_params.mmproj.path.empty()) {
2737+
SRV_INF("loading multimodal projection model '%s'\n", params.llm_params.mmproj.path.c_str());
2738+
2739+
if (params.llm_params.n_ctx < 2048) {
2740+
SRV_WRN("%s", "n_ctx is too small for multimodal projection, setting to 2048\n");
2741+
params.llm_params.n_ctx = 2048;
2742+
}
2743+
// NB(thxCode): clip_context_params is a patch.
2744+
clip_context_params llm_params_clip{
2745+
/* use_gpu */ params.llm_params.n_gpu_layers != 0,
2746+
/* verbosity */ common_log_verbosity_thold > 3 ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_INFO,
2747+
/* max_image_size */ params.max_image_size,
2748+
};
2749+
llm_init_clip = clip_init(params.llm_params.mmproj.path.c_str(), llm_params_clip);
2750+
if (llm_init_clip.ctx_a == nullptr && llm_init_clip.ctx_v == nullptr) {
2751+
SRV_ERR("failed to load multimodal project model, '%s'\n", params.llm_params.mmproj.path.c_str());
2752+
return false;
2753+
}
2754+
llm_ctx_clip_v = llm_init_clip.ctx_v;
2755+
llm_ctx_clip_a = llm_init_clip.ctx_a;
2756+
}
2757+
2758+
// load the draft model if needed.
2759+
if (!params.llm_params.speculative.model.path.empty() && params.llm_params.speculative.n_max > 0) {
2760+
SRV_INF("loading draft model '%s'\n", params.llm_params.speculative.model.path.c_str());
2761+
2762+
common_params llm_params_draft = params.llm_params;
2763+
llm_params_draft.n_parallel = params.llm_params.n_threads_http;
2764+
llm_params_draft.embedding = false;
2765+
llm_params_draft.model = params.llm_params.speculative.model;
2766+
llm_params_draft.n_gpu_layers = params.llm_params.speculative.n_gpu_layers;
2767+
llm_params_draft.cpuparams = params.llm_params.speculative.cpuparams;
2768+
llm_params_draft.cpuparams_batch = params.llm_params.speculative.cpuparams_batch;
2769+
llm_params_draft.cache_type_k = GGML_TYPE_F16;
2770+
llm_params_draft.cache_type_v = GGML_TYPE_F16;
2771+
llm_params_draft.warmup = false;
2772+
llm_init_draft = common_init_from_params(llm_params_draft);
2773+
llm_model_draft = llm_init_draft.model.get();
2774+
llm_ctx_draft = llm_init_draft.context.get();
2775+
if (llm_model_draft == nullptr) {
2776+
SRV_ERR("failed to load draft model, '%s'\n", params.llm_params.speculative.model.path.c_str());
2777+
return false;
2778+
}
2779+
llm_vocab_draft = llama_model_get_vocab(llm_model_draft);
2780+
batch_text_draft = llama_batch_init(int32_t(llama_n_ctx(llm_ctx_draft)), 0, 1);
2781+
}
2782+
27352783
common_params llm_params = params.llm_params;
27362784
llm_params.n_parallel = params.llm_params.n_threads_http;
27372785
llm_init = common_init_from_params(llm_params);
@@ -2755,29 +2803,8 @@ struct httpserver {
27552803
batch_text = llama_batch_init(llm_ctx_size, 0, 1);
27562804
batch_text_temp = llama_batch_init(llm_ctx_size, 0, 1);
27572805

2758-
// load multimodal projection model
2759-
if (!params.llm_params.mmproj.path.empty()) {
2760-
SRV_INF("loading multimodal projection model '%s'\n", params.llm_params.mmproj.path.c_str());
2761-
2762-
if (params.llm_params.n_ctx < 2048) {
2763-
SRV_WRN("%s", "n_ctx is too small for multimodal projection, setting to 2048\n");
2764-
params.llm_params.n_ctx = 2048;
2765-
}
2766-
// NB(thxCode): clip_context_params is a patch.
2767-
clip_context_params llm_params_clip{
2768-
/* use_gpu */ params.llm_params.n_gpu_layers != 0,
2769-
/* verbosity */ common_log_verbosity_thold > 3 ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_INFO,
2770-
/* max_image_size */ params.max_image_size,
2771-
};
2772-
llm_init_clip = clip_init(params.llm_params.mmproj.path.c_str(), llm_params_clip);
2773-
if (llm_init_clip.ctx_a == nullptr && llm_init_clip.ctx_v == nullptr) {
2774-
SRV_ERR("failed to load multimodal project model, '%s'\n", params.llm_params.mmproj.path.c_str());
2775-
return false;
2776-
}
2777-
llm_ctx_clip_v = llm_init_clip.ctx_v;
2778-
llm_ctx_clip_a = llm_init_clip.ctx_a;
2779-
2780-
// check multimodal projection model compatibility
2806+
// check multimodal projection model compatibility if needed
2807+
if (llm_ctx_clip_v != nullptr || llm_ctx_clip_a != nullptr) {
27812808
bool discard = false;
27822809
if (llm_ctx_clip_v != nullptr) {
27832810
const int32_t n_embd_clip = clip_n_mmproj_embd(llm_ctx_clip_v);
@@ -2815,31 +2842,8 @@ struct httpserver {
28152842
}
28162843
}
28172844

2818-
// load the draft model if needed
2819-
if (!params.llm_params.speculative.model.path.empty() && params.llm_params.speculative.n_max > 0) {
2820-
SRV_INF("loading draft model '%s'\n", params.llm_params.speculative.model.path.c_str());
2821-
2822-
common_params llm_params_draft = params.llm_params;
2823-
llm_params_draft.n_parallel = params.llm_params.n_threads_http;
2824-
llm_params_draft.embedding = false;
2825-
llm_params_draft.model = params.llm_params.speculative.model;
2826-
llm_params_draft.n_gpu_layers = params.llm_params.speculative.n_gpu_layers;
2827-
llm_params_draft.cpuparams = params.llm_params.speculative.cpuparams;
2828-
llm_params_draft.cpuparams_batch = params.llm_params.speculative.cpuparams_batch;
2829-
llm_params_draft.cache_type_k = GGML_TYPE_F16;
2830-
llm_params_draft.cache_type_v = GGML_TYPE_F16;
2831-
llm_params_draft.warmup = false;
2832-
llm_init_draft = common_init_from_params(llm_params_draft);
2833-
llm_model_draft = llm_init_draft.model.get();
2834-
llm_ctx_draft = llm_init_draft.context.get();
2835-
if (llm_model_draft == nullptr) {
2836-
SRV_ERR("failed to load draft model, '%s'\n", params.llm_params.speculative.model.path.c_str());
2837-
return false;
2838-
}
2839-
llm_vocab_draft = llama_model_get_vocab(llm_model_draft);
2840-
batch_text_draft = llama_batch_init(int32_t(llama_n_ctx(llm_ctx_draft)), 0, 1);
2841-
2842-
// check draft model compatibility if needed
2845+
// check draft model compatibility if needed
2846+
if (llm_model_draft != nullptr) {
28432847
const bool vocab_type_draft = llama_vocab_type(llm_vocab_draft);
28442848
const bool vocab_type = llama_vocab_type(llm_vocab);
28452849
if (vocab_type_draft != vocab_type) {

0 commit comments

Comments
 (0)