Skip to content

Commit bd0e21c

Browse files
jennyf19Jenny Ferries
jennyf19
and
Jenny Ferries
authored
Add chat template kwargs to C API (#1102)
* Add chat template kwargs to C API Preserve the existing API while allowing typed JSON context values for model-specific chat templates. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a9f5bae1-1b88-4311-8fac-6224769e0400 * Add temporary branch validation workflow Run the repository C API build and C++ tests on Jenny's fork while upstream Azure Pipelines await maintainer authorization. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a9f5bae1-1b88-4311-8fac-6224769e0400 * Use supported Minja boolean syntax in test Exercise the same typed false value without relying on the unsupported is false predicate in an explicit template. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a9f5bae1-1b88-4311-8fac-6224769e0400 * Remove temporary fork validation workflow The repository C API build and full C++ test suite passed in fork run 32069739093. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a9f5bae1-1b88-4311-8fac-6224769e0400 * Reject null tokenizer in chat template APIs Return an invalid-argument error instead of dereferencing a null tokenizer, with a focused regression test and temporary fork validation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a9f5bae1-1b88-4311-8fac-6224769e0400 * Reject empty chat template kwargs Keep the public contract strict: callers must pass a JSON object or null, never an empty string. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a9f5bae1-1b88-4311-8fac-6224769e0400 * Remove completed fork validation workflow The strict kwargs validation and null-tokenizer regression pass the full C API C++ suite in run 32070849855. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a9f5bae1-1b88-4311-8fac-6224769e0400 --------- Co-authored-by: Jenny Ferries <jenny.ferries@microsoft.com> Copilot-Session: a9f5bae1-1b88-4311-8fac-6224769e0400
1 parent ed0277e commit bd0e21c

5 files changed

Lines changed: 177 additions & 18 deletions

File tree

‎include/ortx_tokenizer.h‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,30 @@ extError_t ORTX_API_CALL OrtxApplyChatTemplate(const OrtxTokenizer* tokenizer, c
276276
const char* input, const char* tools, OrtxTensorResult** output,
277277
bool add_generation_prompt, bool tokenize);
278278

279+
/**
280+
* @brief Applies a chat template with additional template context values.
281+
*
282+
* Behaves like OrtxApplyChatTemplate, while also adding the properties from
283+
* template_kwargs to the chat template context. template_kwargs must be a
284+
* null-terminated JSON object or null. Core context properties such as messages,
285+
* tools, and add_generation_prompt cannot be overridden.
286+
*
287+
* @param tokenizer Pointer to an OrtxTokenizer used for template processing.
288+
* @param template_str Null-terminated string representing the chat template; can be null if tokenizer.json has one.
289+
* @param input Null-terminated string containing the input to be processed.
290+
* @param tools Null-terminated string containing the function tools.
291+
* @param template_kwargs Null-terminated JSON object containing additional template context values; can be null.
292+
* @param output Pointer to an OrtxTensorResult that will be populated with the output strings,
293+
* if tokenize is true, the ids will be in the output as indexed 1.
294+
* @param add_generation_prompt Indicates whether to add a generation prompt to the output.
295+
* @param tokenize Indicates whether to tokenize the templated text to IDs.
296+
* @return extError_t Returns an error code indicating success or the type of failure.
297+
*/
298+
extError_t ORTX_API_CALL OrtxApplyChatTemplateWithOptions(const OrtxTokenizer* tokenizer, const char* template_str,
299+
const char* input, const char* tools,
300+
const char* template_kwargs, OrtxTensorResult** output,
301+
bool add_generation_prompt, bool tokenize);
302+
279303
#ifdef __cplusplus
280304
}
281305
#endif

‎shared/api/c_api_tokenizer.cc‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -467,8 +467,16 @@ extError_t ORTX_API_CALL OrtxApplyChatTemplate(const OrtxTokenizer* tokenizer, c
467467
const char* input, const char* tools,
468468
OrtxTensorResult** output, bool add_generation_prompt,
469469
bool tokenize) {
470-
if (tokenizer == nullptr && template_str == nullptr) {
471-
ReturnableStatus::last_error_message_ = "both tokenizer and template_str are null, no template to apply";
470+
return OrtxApplyChatTemplateWithOptions(tokenizer, template_str, input, tools, nullptr, output,
471+
add_generation_prompt, tokenize);
472+
}
473+
474+
extError_t ORTX_API_CALL OrtxApplyChatTemplateWithOptions(const OrtxTokenizer* tokenizer, const char* template_str,
475+
const char* input, const char* tools,
476+
const char* template_kwargs, OrtxTensorResult** output,
477+
bool add_generation_prompt, bool tokenize) {
478+
if (tokenizer == nullptr) {
479+
ReturnableStatus::last_error_message_ = "tokenizer is null";
472480
return kOrtxErrorInvalidArgument;
473481
}
474482

@@ -485,7 +493,8 @@ extError_t ORTX_API_CALL OrtxApplyChatTemplate(const OrtxTokenizer* tokenizer, c
485493

486494
std::string text;
487495
std::vector<extTokenId_t> ids_vec;
488-
status = token_ptr->ApplyChatTemplate(template_str, input, tools, text, ids_vec, add_generation_prompt, tokenize);
496+
status = token_ptr->ApplyChatTemplate(template_str, input, tools, template_kwargs, text, ids_vec,
497+
add_generation_prompt, tokenize);
489498
if (status.IsOk()) {
490499
auto result = std::make_unique<ort_extensions::TensorResult>();
491500
std::vector<std::unique_ptr<ortc::TensorBase>> tensors;

‎shared/api/chat_template.cc‎

Lines changed: 25 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -376,8 +376,9 @@ std::string normalize_tool_quotes(const std::string& input) {
376376
}
377377

378378
OrtxStatus TokenizerImpl::ApplyChatTemplate(const char* template_str, const char* message, const char* tools,
379-
std::string& output, std::vector<extTokenId_t>& ids_vec,
380-
bool add_generation_prompt, bool tokenize) const {
379+
const char* template_kwargs, std::string& output,
380+
std::vector<extTokenId_t>& ids_vec, bool add_generation_prompt,
381+
bool tokenize) const {
381382
OrtxStatus status;
382383
std::string input_str = minja::normalize_newlines(message);
383384

@@ -408,7 +409,21 @@ OrtxStatus TokenizerImpl::ApplyChatTemplate(const char* template_str, const char
408409
throw std::runtime_error("Invalid or unsupported chat template.");
409410
}
410411

411-
std::shared_ptr<minja::Context> context;
412+
json context_values = json::object();
413+
if (template_kwargs) {
414+
if (*template_kwargs == '\0') {
415+
throw std::runtime_error("template_kwargs must be a JSON object or null.");
416+
}
417+
auto parsed_kwargs = json::parse(minja::normalize_newlines(template_kwargs), nullptr,
418+
/*allow_exceptions=*/false);
419+
if (parsed_kwargs.is_discarded()) {
420+
throw std::runtime_error("Invalid template_kwargs JSON.");
421+
}
422+
if (!parsed_kwargs.is_object()) {
423+
throw std::runtime_error("template_kwargs must be a JSON object.");
424+
}
425+
context_values = std::move(parsed_kwargs);
426+
}
412427

413428
// Check Phi-4-mini tool call case for quote normalization
414429
bool phi_4_mini = false;
@@ -462,20 +477,16 @@ OrtxStatus TokenizerImpl::ApplyChatTemplate(const char* template_str, const char
462477
tools_json = NormalizeTools(tools_str.c_str());
463478
}
464479

465-
// Add tools to the context
466-
context = minja::Context::make(json({
467-
{"messages", actual_messages},
468-
{"tools", tools_json},
469-
{"add_generation_prompt", add_generation_prompt},
470-
}));
480+
context_values["tools"] = std::move(tools_json);
471481
} else {
472-
// No tools input, just use the messages
473-
context = minja::Context::make(json({
474-
{"messages", actual_messages},
475-
{"add_generation_prompt", add_generation_prompt},
476-
}));
482+
context_values.erase("tools");
477483
}
478484

485+
// Core request values take precedence over additional template kwargs.
486+
context_values["messages"] = std::move(actual_messages);
487+
context_values["add_generation_prompt"] = add_generation_prompt;
488+
auto context = minja::Context::make(std::move(context_values));
489+
479490
// Set required context values
480491
context->set("strftime_now", minja::Value::callable(strftime_function));
481492
context->set("bos_token", tok_config_->bos_token_);

‎shared/api/tokenizer_impl.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,8 @@ class TokenizerImpl : public OrtxObjectImpl {
8989
OrtxStatus Id2Token(extTokenId_t id, std::string& token, TokenizerDecodingState** state, bool skip_special_tokens) const;
9090
OrtxStatus GetDecoderPromptIds(size_t batch_size, const char* lang, const char* task, int no_timestamps,
9191
std::vector<std::vector<extTokenId_t>>& t_ids) const;
92-
OrtxStatus ApplyChatTemplate(const char* template_str, const char* message, const char* tools, std::string& output,
92+
OrtxStatus ApplyChatTemplate(const char* template_str, const char* message, const char* tools,
93+
const char* template_kwargs, std::string& output,
9394
std::vector<extTokenId_t>& ids_vec, bool add_generation_prompt, bool tokenize) const;
9495

9596
private:

‎test/pp_api_test/test_tokenizer_chat.cc‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2255,4 +2255,118 @@ TEST(OrtxTokenizerTest, ChatTemplateDivisionByZero) {
22552255
messages_json.c_str(), nullptr, result.ToBeAssigned(), false, false);
22562256
EXPECT_NE(err, kOrtxOK) << "Expected modulo by zero to return an error.";
22572257
}
2258+
}
2259+
2260+
TEST(OrtxTokenizerTest, ChatTemplateAcceptsTypedTemplateKwargs) {
2261+
OrtxObjectPtr<OrtxTokenizer> tokenizer(OrtxCreateTokenizer, "data/phi-4-base");
2262+
ASSERT_EQ(tokenizer.Code(), kOrtxOK) << OrtxGetLastErrorMessage();
2263+
2264+
const std::string template_str =
2265+
R"({% if enable_thinking is defined and not enable_thinking %}NO_THINK{% else %}THINK{% endif %}|{{ reasoning_effort }}|{{ level }})";
2266+
const std::string messages_json = R"([{"role":"user","content":"Hello"}])";
2267+
const std::string template_kwargs =
2268+
R"({"enable_thinking":false,"reasoning_effort":"low","level":2})";
2269+
OrtxObjectPtr<OrtxTensorResult> result;
2270+
2271+
auto err = OrtxApplyChatTemplateWithOptions(
2272+
tokenizer.get(), template_str.c_str(), messages_json.c_str(), nullptr,
2273+
template_kwargs.c_str(), result.ToBeAssigned(), true, false);
2274+
ASSERT_EQ(err, kOrtxOK) << OrtxGetLastErrorMessage();
2275+
2276+
OrtxObjectPtr<OrtxTensor> tensor;
2277+
ASSERT_EQ(OrtxTensorResultGetAt(result.get(), 0, tensor.ToBeAssigned()), kOrtxOK);
2278+
const char* text = nullptr;
2279+
ASSERT_EQ(OrtxGetTensorData(tensor.get(), reinterpret_cast<const void**>(&text), nullptr, nullptr), kOrtxOK);
2280+
EXPECT_STREQ(text, "NO_THINK|low|2");
2281+
}
2282+
2283+
TEST(OrtxTokenizerTest, ChatTemplateKwargsCannotOverrideCoreContext) {
2284+
OrtxObjectPtr<OrtxTokenizer> tokenizer(OrtxCreateTokenizer, "data/phi-4-base");
2285+
ASSERT_EQ(tokenizer.Code(), kOrtxOK) << OrtxGetLastErrorMessage();
2286+
2287+
const std::string template_str =
2288+
R"({{ messages[0].content }}|{% if add_generation_prompt %}GEN{% else %}NO_GEN{% endif %}|{% if tools is defined %}TOOLS{% else %}NO_TOOLS{% endif %})";
2289+
const std::string messages_json = R"([{"role":"user","content":"Hello"}])";
2290+
const std::string template_kwargs =
2291+
R"({"messages":[{"role":"user","content":"Override"}],"add_generation_prompt":false,"tools":[{"name":"override"}]})";
2292+
OrtxObjectPtr<OrtxTensorResult> result;
2293+
2294+
auto err = OrtxApplyChatTemplateWithOptions(
2295+
tokenizer.get(), template_str.c_str(), messages_json.c_str(), nullptr,
2296+
template_kwargs.c_str(), result.ToBeAssigned(), true, false);
2297+
ASSERT_EQ(err, kOrtxOK) << OrtxGetLastErrorMessage();
2298+
2299+
OrtxObjectPtr<OrtxTensor> tensor;
2300+
ASSERT_EQ(OrtxTensorResultGetAt(result.get(), 0, tensor.ToBeAssigned()), kOrtxOK);
2301+
const char* text = nullptr;
2302+
ASSERT_EQ(OrtxGetTensorData(tensor.get(), reinterpret_cast<const void**>(&text), nullptr, nullptr), kOrtxOK);
2303+
EXPECT_STREQ(text, "Hello|GEN|NO_TOOLS");
2304+
}
2305+
2306+
TEST(OrtxTokenizerTest, ChatTemplateRejectsInvalidTemplateKwargs) {
2307+
OrtxObjectPtr<OrtxTokenizer> tokenizer(OrtxCreateTokenizer, "data/phi-4-base");
2308+
ASSERT_EQ(tokenizer.Code(), kOrtxOK) << OrtxGetLastErrorMessage();
2309+
2310+
const std::string messages_json = R"([{"role":"user","content":"Hello"}])";
2311+
OrtxObjectPtr<OrtxTensorResult> result;
2312+
2313+
auto empty_string = OrtxApplyChatTemplateWithOptions(
2314+
tokenizer.get(), "{{ messages[0].content }}", messages_json.c_str(), nullptr,
2315+
"", result.ToBeAssigned(), true, false);
2316+
EXPECT_EQ(empty_string, kOrtxErrorInvalidArgument);
2317+
EXPECT_STREQ(OrtxGetLastErrorMessage(), "template_kwargs must be a JSON object or null.");
2318+
2319+
auto invalid_json = OrtxApplyChatTemplateWithOptions(
2320+
tokenizer.get(), "{{ messages[0].content }}", messages_json.c_str(), nullptr,
2321+
"{", result.ToBeAssigned(), true, false);
2322+
EXPECT_EQ(invalid_json, kOrtxErrorInvalidArgument);
2323+
EXPECT_STREQ(OrtxGetLastErrorMessage(), "Invalid template_kwargs JSON.");
2324+
2325+
auto non_object = OrtxApplyChatTemplateWithOptions(
2326+
tokenizer.get(), "{{ messages[0].content }}", messages_json.c_str(), nullptr,
2327+
"[]", result.ToBeAssigned(), true, false);
2328+
EXPECT_EQ(non_object, kOrtxErrorInvalidArgument);
2329+
EXPECT_STREQ(OrtxGetLastErrorMessage(), "template_kwargs must be a JSON object.");
2330+
}
2331+
2332+
TEST(OrtxTokenizerTest, ChatTemplateRejectsNullTokenizerWithExplicitTemplate) {
2333+
const std::string messages_json = R"([{"role":"user","content":"Hello"}])";
2334+
OrtxObjectPtr<OrtxTensorResult> result;
2335+
2336+
auto err = OrtxApplyChatTemplateWithOptions(
2337+
nullptr, "{{ messages[0].content }}", messages_json.c_str(), nullptr,
2338+
nullptr, result.ToBeAssigned(), true, false);
2339+
EXPECT_EQ(err, kOrtxErrorInvalidArgument);
2340+
EXPECT_STREQ(OrtxGetLastErrorMessage(), "tokenizer is null");
2341+
}
2342+
2343+
TEST(OrtxTokenizerTest, LegacyChatTemplateApiMatchesNullTemplateKwargs) {
2344+
OrtxObjectPtr<OrtxTokenizer> tokenizer(OrtxCreateTokenizer, "data/phi-4-base");
2345+
ASSERT_EQ(tokenizer.Code(), kOrtxOK) << OrtxGetLastErrorMessage();
2346+
2347+
const std::string template_str = R"({{ messages[0].content }}|{{ add_generation_prompt }})";
2348+
const std::string messages_json = R"([{"role":"user","content":"Hello"}])";
2349+
OrtxObjectPtr<OrtxTensorResult> legacy_result;
2350+
OrtxObjectPtr<OrtxTensorResult> options_result;
2351+
2352+
ASSERT_EQ(OrtxApplyChatTemplate(
2353+
tokenizer.get(), template_str.c_str(), messages_json.c_str(), nullptr,
2354+
legacy_result.ToBeAssigned(), true, false),
2355+
kOrtxOK);
2356+
ASSERT_EQ(OrtxApplyChatTemplateWithOptions(
2357+
tokenizer.get(), template_str.c_str(), messages_json.c_str(), nullptr, nullptr,
2358+
options_result.ToBeAssigned(), true, false),
2359+
kOrtxOK);
2360+
2361+
OrtxObjectPtr<OrtxTensor> legacy_tensor;
2362+
OrtxObjectPtr<OrtxTensor> options_tensor;
2363+
ASSERT_EQ(OrtxTensorResultGetAt(legacy_result.get(), 0, legacy_tensor.ToBeAssigned()), kOrtxOK);
2364+
ASSERT_EQ(OrtxTensorResultGetAt(options_result.get(), 0, options_tensor.ToBeAssigned()), kOrtxOK);
2365+
const char* legacy_text = nullptr;
2366+
const char* options_text = nullptr;
2367+
ASSERT_EQ(OrtxGetTensorData(legacy_tensor.get(), reinterpret_cast<const void**>(&legacy_text), nullptr, nullptr),
2368+
kOrtxOK);
2369+
ASSERT_EQ(OrtxGetTensorData(options_tensor.get(), reinterpret_cast<const void**>(&options_text), nullptr, nullptr),
2370+
kOrtxOK);
2371+
EXPECT_STREQ(legacy_text, options_text);
22582372
}

0 commit comments

Comments
 (0)