server : accumulate generated text and tokens as parse input (#29876)
* server : collect text and token input * common : add a tokenize helper that aligns tokens with bytes * common : simplify tokenize logic
This commit is contained in:
1 parent
448147d42a
commit
18b5f8b186
8 files changed
+149
-58
No files matched your search
@@ -76,7 +76,7 @@ json task_params::to_json(bool only_metrics) const {
|
||||
{"chat_format", common_chat_format_name(chat_parser_params.format)},
|
||||
{"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)},
|
||||
{"reasoning_in_content", chat_parser_params.reasoning_in_content},
|
||||
{"generation_prompt", chat_parser_params.generation_prompt},
|
||||
{"generation_prompt", chat_parser_params.generation_prompt.text},
|
||||
{"samplers", samplers},
|
||||
{"speculative.types", common_speculative_type_name_str(speculative.types)},
|
||||
{"timings_per_token", timings_per_token},
|
||||
@@ -135,7 +135,7 @@ json task_params::to_json(bool only_metrics) const {
|
||||
{"chat_format", common_chat_format_name(chat_parser_params.format)},
|
||||
{"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)},
|
||||
{"reasoning_in_content", chat_parser_params.reasoning_in_content},
|
||||
{"generation_prompt", chat_parser_params.generation_prompt},
|
||||
{"generation_prompt", chat_parser_params.generation_prompt.text},
|
||||
{"samplers", samplers},
|
||||
{"speculative.types", common_speculative_type_name_str(speculative.types)},
|
||||
{"timings_per_token", timings_per_token},
|
||||
@@ -155,20 +155,20 @@ task_result_state::task_result_state(const common_chat_parser_params & chat_pars
|
||||
, oai_resp_message_id("msg_" + random_string()) {
|
||||
if (chat_parser_params.is_continuation && !chat_parser_params.echo) {
|
||||
// initialize chat_msg to avoid emitting a delta containing the assistant prefill
|
||||
chat_msg = common_chat_parse("", true, chat_parser_params);
|
||||
chat_msg = common_chat_parse(generated_input, true, chat_parser_params);
|
||||
}
|
||||
}
|
||||
|
||||
common_chat_msg task_result_state::update_chat_msg(
|
||||
const std::string & text_added,
|
||||
const common_chat_input & added,
|
||||
bool is_partial,
|
||||
std::vector<common_chat_msg_diff> & diffs,
|
||||
bool filter_tool_calls) {
|
||||
generated_text += text_added;
|
||||
generated_input.append(added);
|
||||
auto msg_prv_copy = chat_msg;
|
||||
//SRV_DBG("Parsing chat message: %s\n", generated_text.c_str());
|
||||
//SRV_DBG("Parsing chat message: %s\n", generated_input.text.c_str());
|
||||
auto new_msg = common_chat_parse(
|
||||
generated_text,
|
||||
generated_input,
|
||||
is_partial,
|
||||
chat_parser_params);
|
||||
if (!new_msg.empty()) {
|
||||
@@ -340,7 +340,7 @@ json server_task_result_cmpl_final::to_json() {
|
||||
json server_task_result_cmpl_final::to_json_non_oaicompat() {
|
||||
json res = json {
|
||||
{"index", index},
|
||||
{"content", content},
|
||||
{"content", content.text},
|
||||
{"tokens", tokens},
|
||||
{"id_slot", id_slot},
|
||||
{"stop", true},
|
||||
@@ -386,7 +386,7 @@ json server_task_result_cmpl_final::to_json_oaicompat() {
|
||||
json res = json {
|
||||
{"choices", json::array({
|
||||
json{
|
||||
{"text", content},
|
||||
{"text", content.text},
|
||||
{"index", index},
|
||||
{"logprobs", logprobs},
|
||||
{"finish_reason", finish_reason},
|
||||
@@ -418,7 +418,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat() {
|
||||
msg = oaicompat_msg;
|
||||
} else {
|
||||
msg.role = "assistant";
|
||||
msg.content = content;
|
||||
msg.content = content.text;
|
||||
}
|
||||
if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {
|
||||
finish_reason = msg.tool_calls.empty() ? "stop" : "tool_calls";
|
||||
@@ -531,7 +531,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp() {
|
||||
msg = oaicompat_msg;
|
||||
} else {
|
||||
msg.role = "assistant";
|
||||
msg.content = content;
|
||||
msg.content = content.text;
|
||||
}
|
||||
|
||||
std::vector<json> output;
|
||||
@@ -741,7 +741,7 @@ json server_task_result_cmpl_final::to_json_anthropic() {
|
||||
msg = oaicompat_msg;
|
||||
} else {
|
||||
msg.role = "assistant";
|
||||
msg.content = content;
|
||||
msg.content = content.text;
|
||||
}
|
||||
|
||||
// thinking block comes first (Anthropic extended thinking format)
|
||||
@@ -1050,7 +1050,7 @@ json server_task_result_cmpl_partial::to_json_non_oaicompat() {
|
||||
// non-OAI-compat JSON
|
||||
json res = json {
|
||||
{"index", index},
|
||||
{"content", content},
|
||||
{"content", content.text},
|
||||
{"tokens", tokens},
|
||||
{"stop", false},
|
||||
{"id_slot", id_slot},
|
||||
@@ -1081,7 +1081,7 @@ json server_task_result_cmpl_partial::to_json_oaicompat() {
|
||||
json res = json {
|
||||
{"choices", json::array({
|
||||
json{
|
||||
{"text", content},
|
||||
{"text", content.text},
|
||||
{"index", index},
|
||||
{"logprobs", logprobs},
|
||||
{"finish_reason", nullptr},
|
||||
@@ -1316,7 +1316,7 @@ json server_task_result_cmpl_partial::to_json_oaicompat_resp() {
|
||||
json server_task_result_cmpl_partial::to_json_oaicompat_asr() {
|
||||
json event = json {
|
||||
{"type", "transcript.text.delta"},
|
||||
{"delta", content},
|
||||
{"delta", content.text},
|
||||
};
|
||||
return event;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user