server : accumulate generated text and tokens as parse input (#29876)

* server : collect text and token input

* common : add a tokenize helper that aligns tokens with bytes

* common : simplify tokenize logic
This commit is contained in:
Aldehir Rojas authored and GitHub committed 2026-10-07 11:49:17 -05:00
1 parent 448147d42a
commit 18b5f8b186
8 files changed
+149 -58

No files matched your search

+7 -1
View File
@@ -311,8 +311,14 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
->set_desc("Generation prompt appended to the chat template output")
->set_handler([&](field_eval_context & ctx, const json & data) {
std::string s = data.at("generation_prompt").get<std::string>();
ctx.params.chat_parser_params.generation_prompt = s;
ctx.params.sampling.generation_prompt = s;
if (ctx.vocab == nullptr) {
ctx.params.chat_parser_params.generation_prompt = common_chat_input(s);
return;
}
ctx.params.chat_parser_params.generation_prompt = common_chat_input_tokenize(ctx.vocab, s);
}));
add((new field_bool("parse_tool_calls", params.chat_parser_params.parse_tool_calls))