server : accumulate generated text and tokens as parse input (#29876)

* server : collect text and token input

* common : add a tokenize helper that aligns tokens with bytes

* common : simplify tokenize logic
This commit is contained in:
Aldehir Rojas authored and GitHub committed 2026-10-07 11:49:17 -05:00
1 parent 448147d42a
commit 18b5f8b186
8 files changed
+149 -58

No files matched your search

+15 -15
View File
@@ -76,7 +76,7 @@ json task_params::to_json(bool only_metrics) const {
{"chat_format", common_chat_format_name(chat_parser_params.format)},
{"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)},
{"reasoning_in_content", chat_parser_params.reasoning_in_content},
{"generation_prompt", chat_parser_params.generation_prompt},
{"generation_prompt", chat_parser_params.generation_prompt.text},
{"samplers", samplers},
{"speculative.types", common_speculative_type_name_str(speculative.types)},
{"timings_per_token", timings_per_token},
@@ -135,7 +135,7 @@ json task_params::to_json(bool only_metrics) const {
{"chat_format", common_chat_format_name(chat_parser_params.format)},
{"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)},
{"reasoning_in_content", chat_parser_params.reasoning_in_content},
{"generation_prompt", chat_parser_params.generation_prompt},
{"generation_prompt", chat_parser_params.generation_prompt.text},
{"samplers", samplers},
{"speculative.types", common_speculative_type_name_str(speculative.types)},
{"timings_per_token", timings_per_token},
@@ -155,20 +155,20 @@ task_result_state::task_result_state(const common_chat_parser_params & chat_pars
, oai_resp_message_id("msg_" + random_string()) {
if (chat_parser_params.is_continuation && !chat_parser_params.echo) {
// initialize chat_msg to avoid emitting a delta containing the assistant prefill
chat_msg = common_chat_parse("", true, chat_parser_params);
chat_msg = common_chat_parse(generated_input, true, chat_parser_params);
}
}
common_chat_msg task_result_state::update_chat_msg(
const std::string & text_added,
const common_chat_input & added,
bool is_partial,
std::vector<common_chat_msg_diff> & diffs,
bool filter_tool_calls) {
generated_text += text_added;
generated_input.append(added);
auto msg_prv_copy = chat_msg;
//SRV_DBG("Parsing chat message: %s\n", generated_text.c_str());
//SRV_DBG("Parsing chat message: %s\n", generated_input.text.c_str());
auto new_msg = common_chat_parse(
generated_text,
generated_input,
is_partial,
chat_parser_params);
if (!new_msg.empty()) {
@@ -340,7 +340,7 @@ json server_task_result_cmpl_final::to_json() {
json server_task_result_cmpl_final::to_json_non_oaicompat() {
json res = json {
{"index", index},
{"content", content},
{"content", content.text},
{"tokens", tokens},
{"id_slot", id_slot},
{"stop", true},
@@ -386,7 +386,7 @@ json server_task_result_cmpl_final::to_json_oaicompat() {
json res = json {
{"choices", json::array({
json{
{"text", content},
{"text", content.text},
{"index", index},
{"logprobs", logprobs},
{"finish_reason", finish_reason},
@@ -418,7 +418,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat() {
msg = oaicompat_msg;
} else {
msg.role = "assistant";
msg.content = content;
msg.content = content.text;
}
if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {
finish_reason = msg.tool_calls.empty() ? "stop" : "tool_calls";
@@ -531,7 +531,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp() {
msg = oaicompat_msg;
} else {
msg.role = "assistant";
msg.content = content;
msg.content = content.text;
}
std::vector<json> output;
@@ -741,7 +741,7 @@ json server_task_result_cmpl_final::to_json_anthropic() {
msg = oaicompat_msg;
} else {
msg.role = "assistant";
msg.content = content;
msg.content = content.text;
}
// thinking block comes first (Anthropic extended thinking format)
@@ -1050,7 +1050,7 @@ json server_task_result_cmpl_partial::to_json_non_oaicompat() {
// non-OAI-compat JSON
json res = json {
{"index", index},
{"content", content},
{"content", content.text},
{"tokens", tokens},
{"stop", false},
{"id_slot", id_slot},
@@ -1081,7 +1081,7 @@ json server_task_result_cmpl_partial::to_json_oaicompat() {
json res = json {
{"choices", json::array({
json{
{"text", content},
{"text", content.text},
{"index", index},
{"logprobs", logprobs},
{"finish_reason", nullptr},
@@ -1316,7 +1316,7 @@ json server_task_result_cmpl_partial::to_json_oaicompat_resp() {
json server_task_result_cmpl_partial::to_json_oaicompat_asr() {
json event = json {
{"type", "transcript.text.delta"},
{"delta", content},
{"delta", content.text},
};
return event;
}