Skip to content

Commit 888251a

Browse files
radareradare
committed
Improve kvcache misses and improve token usage
1 parent af6153a commit 888251a

6 files changed

Lines changed: 63 additions & 0 deletions

File tree

‎src/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ These can be set with `r2ai -e <keyname>=<value>`
9696
| r2ai.model | Model name. List possibilities with `r2ai -e r2ai.model=?` |
9797
| r2ai.baseurl | Remote LLM base URL. Specify host if necessary e.g http://127.0.0.1:11434. |
9898
| r2ai.apitype | Ollama API endpoint type to use: `chat` (default) or `generate`. |
99+
| r2ai.cacheck | Warn when chat requests are not append-only and may miss provider prompt-cache hits. |
99100
| r2ai.max_tokens | Maximum output tokens or maximum total tokens. Check the appropriate limits for your model |
100101
| r2ai.temperature | How creative the model should be. 0=not creative, 1=very creative |
101102
| r2ai.cmds | R2 command to issue and send output in context to model |

‎src/auto.c‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -388,6 +388,7 @@ R_IPI void cmd_r2ai_a(RCorePluginSession *cps, const char *user_query) {
388388
// If this is the first message in a new conversation, clear previous history
389389
if (r_list_empty (messages) || r_config_get_b (core->config, "r2ai.auto.reset_on_query")) {
390390
r2ai_msgs_clear (messages);
391+
R_FREE (state->cache_prefix);
391392
}
392393

393394
// Add user query
@@ -590,6 +591,7 @@ static void process_conversation_with_llm(RCorePluginSession *cps, bool compact)
590591
if (compact) {
591592
// Clear the conversation and add summary as system message
592593
r2ai_msgs_clear (messages);
594+
R_FREE (state->cache_prefix);
593595
R2AI_Message summary_msg = {
594596
.role = "system",
595597
.content = res

‎src/openai.c‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,54 @@ static char *ollama_generate_prompt_from_messages(const RList *msgs, char **syst
5757
return prompt;
5858
}
5959

60+
static void cache_part(RStrBuf *sb, const char *s) {
61+
const char *v = r_str_get (s);
62+
r_strbuf_appendf (sb, "%zu:%s\n", strlen (v), v);
63+
}
64+
65+
static char *cache_prefix(const char *provider, const char *model, const char *tools_json, const char *messages_json) {
66+
RStrBuf *sb = r_strbuf_new ("");
67+
cache_part (sb, provider);
68+
cache_part (sb, model);
69+
cache_part (sb, tools_json);
70+
const char *messages = r_str_get (messages_json);
71+
size_t len = strlen (messages);
72+
if (len > 0 && messages[len - 1] == ']') {
73+
len--;
74+
}
75+
r_strbuf_appendf (sb, "%zu:", len);
76+
r_strbuf_append_n (sb, messages, len);
77+
return r_strbuf_drain (sb);
78+
}
79+
80+
static size_t prefix_len(const char *a, const char *b) {
81+
size_t i = 0;
82+
while (a[i] && b[i] && a[i] == b[i]) {
83+
i++;
84+
}
85+
return i;
86+
}
87+
88+
static void check_chat_cache(RCorePluginSession *cps, const char *provider, const char *model, const char *tools_json, const char *messages_json) {
89+
RCore *core = cps->core;
90+
R2AI_State *state = cps->data;
91+
if (!state || !r_config_get_b (core->config, "r2ai.cacheck")) {
92+
return;
93+
}
94+
char *prefix = cache_prefix (provider, model, tools_json, messages_json);
95+
if (state->cache_prefix) {
96+
const size_t old_len = strlen (state->cache_prefix);
97+
if (!r_str_startswith (prefix, state->cache_prefix)) {
98+
const size_t kept = prefix_len (state->cache_prefix, prefix);
99+
R_LOG_WARN ("Chat cache prefix changed: preserved %zu/%zu bytes from previous request. Keep system prompt, tool catalog, and previous messages append-only to maximize provider cache hits.", kept, old_len);
100+
} else {
101+
R_LOG_DEBUG ("Chat cache prefix preserved: %zu bytes", old_len);
102+
}
103+
R_FREE (state->cache_prefix);
104+
}
105+
state->cache_prefix = prefix;
106+
}
107+
60108
R_IPI R2AI_ChatResponse *r2ai_openai(RCorePluginSession *cps, R2AIArgs args) {
61109
RCore *core = cps->core;
62110
const char *provider_name = R_STR_ISNOTEMPTY (args.provider)
@@ -229,6 +277,9 @@ R_IPI R2AI_ChatResponse *r2ai_openai(RCorePluginSession *cps, R2AIArgs args) {
229277
pj_end (pj);
230278

231279
char *complete_json = pj_drain (pj);
280+
if (!use_generate) {
281+
check_chat_cache (cps, provider_name, model_name, openai_tools_json, chat_messages_json);
282+
}
232283
free (chat_messages_json);
233284
free (generate_prompt);
234285
free (generate_system);

‎src/r2ai.1‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,9 @@ Remote LLM base URL (e.g., http://127.0.0.1:11434).
9191
.B r2ai.apitype
9292
Ollama API endpoint type to use: chat (default) or generate.
9393
.TP
94+
.B r2ai.cacheck
95+
Warn when chat requests are not append-only and may miss provider prompt-cache hits.
96+
.TP
9497
.B r2ai.max_tokens
9598
Maximum output tokens or total tokens.
9699
.TP

‎src/r2ai.c‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -617,6 +617,7 @@ R_API void cmd_r2ai(RCorePluginSession *cps, const char *input) {
617617
r_cons_printf (core->cons, "No conversation history to reset\n");
618618
} else {
619619
r2ai_msgs_clear (messages);
620+
R_FREE (state->cache_prefix);
620621
r_cons_printf (core->cons, "Chat conversation context has been reset\n");
621622
}
622623
} else if (r_str_startswith (input, "-m")) {
@@ -797,6 +798,8 @@ R_IPI bool r2ai_init(RCorePluginSession *cps) {
797798
r_config_desc (core->config, "r2ai.baseurl", "Base URL for provider API (overrides default endpoints)");
798799
r_config_set_cb (core->config, "r2ai.apitype", "chat", &cb_r2ai_apitype);
799800
r_config_desc (core->config, "r2ai.apitype", "Ollama API endpoint type to use (chat or generate)");
801+
r_config_set_b (core->config, "r2ai.cacheck", false);
802+
r_config_desc (core->config, "r2ai.cacheck", "Warn when chat requests are not append-only and may miss provider prompt-cache hits");
800803
r_config_set_i (core->config, "r2ai.max_tokens", 4096); // max output tokens, or max total tokens
801804
r_config_desc (core->config, "r2ai.max_tokens", "Maximum tokens for LLM responses (output/total depending on provider)");
802805
r_config_set_i (core->config, "r2ai.thinking_tokens", 0);
@@ -890,6 +893,7 @@ R_API bool r2ai_fini(RCorePluginSession *cps) {
890893
r_config_rm (core->config, "r2ai.cmds");
891894
r_config_rm (core->config, "r2ai.model");
892895
r_config_rm (core->config, "r2ai.apitype");
896+
r_config_rm (core->config, "r2ai.cacheck");
893897
r_config_rm (core->config, "r2ai.prompt");
894898
r_config_rm (core->config, "r2ai.stream");
895899
r_config_rm (core->config, "r2ai.system");
@@ -919,6 +923,7 @@ R_API bool r2ai_fini(RCorePluginSession *cps) {
919923
state->prompt_auto = NULL;
920924
free (state->vertex_token);
921925
state->vertex_token = NULL;
926+
R_FREE (state->cache_prefix);
922927
free (state);
923928
cps->data = NULL;
924929
}

‎src/r2ai.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,7 @@ typedef struct r2ai_state_t {
173173
char *prompt_auto; // Auto-mode system prompt
174174
char *vertex_token; // Cached Vertex AI OAuth2 token
175175
ut64 vertex_token_expiry; // Monotonic microseconds when the cached token expires
176+
char *cache_prefix; // Last chat request prefix used by r2ai.cacheck
176177
} R2AI_State;
177178

178179
// conversation

0 commit comments

Comments
 (0)