server : accumulate generated text and tokens as parse input (#29876)

* server : collect text and token input

* common : add a tokenize helper that aligns tokens with bytes

* common : simplify tokenize logic
This commit is contained in:
Aldehir Rojas authored and GitHub committed 2026-10-07 11:49:17 -05:00
1 parent 448147d42a
commit 18b5f8b186
8 files changed
+149 -58

No files matched your search

+64 -9
View File
@@ -1452,14 +1452,70 @@ common_chat_params common_chat_templates_apply(const struct common_chat_template
common_chat_templates_apply_legacy(tmpls, inputs);
}
common_chat_msg common_chat_parse(const std::string & input,
void common_chat_input::append(const std::string & piece, llama_token token) {
if (piece.empty()) {
return;
}
tokens.push_back(token);
tokens.resize(tokens.size() + piece.size() - 1, LLAMA_TOKEN_NULL);
text += piece;
}
void common_chat_input::append(const common_chat_input & chunk) {
tokens.insert(tokens.end(), chunk.tokens.begin(), chunk.tokens.end());
text += chunk.text;
}
void common_chat_input::truncate(size_t pos) {
if (pos < text.size()) {
text.erase(pos);
tokens.resize(pos);
}
}
common_chat_input common_chat_input::substr(size_t pos, size_t n) const {
common_chat_input out;
out.text = text.substr(pos, n);
out.tokens.assign(tokens.begin() + pos, tokens.begin() + pos + out.size());
return out;
}
void common_chat_input::prepend(const std::string & prefix) {
tokens.insert(tokens.begin(), prefix.size(), LLAMA_TOKEN_NULL);
text = prefix + text;
}
void common_chat_input::prepend(const common_chat_input & prefix) {
tokens.insert(tokens.begin(), prefix.tokens.begin(), prefix.tokens.end());
text = prefix.text + text;
}
common_chat_input common_chat_input_tokenize(const llama_vocab * vocab, const std::string & text) {
common_chat_input input;
auto tokens = common_tokenize(vocab, text, false, true);
for (size_t i = 0; i < tokens.size(); i++) {
std::string piece = common_token_to_piece(vocab, tokens[i], true);
if (i == 0 && std::isspace(piece[0]) && !std::isspace(text[0])) {
// Some tokenizers will add a space before the first special token, need to exclude
continue;
}
input.append(piece, tokens[i]);
}
if (input.text != text) {
// the pieces do not give back the same text, keep the text without tokens
return common_chat_input(text);
}
return input;
}
common_chat_msg common_chat_parse(const common_chat_input & input,
bool is_partial,
const common_chat_parser_params & params) {
return common_chat_peg_parse(params.parser, input, is_partial, params);
}
common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser,
const std::string & input,
const common_chat_input & input,
bool is_partial,
const common_chat_parser_params & params) {
const common_peg_arena & parser = src_parser.empty() ?
@@ -1470,18 +1526,17 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars
LOG_DBG("No parser definition detected, assuming pure content parser.");
}
const std::string effective_input = params.generation_prompt.empty()
? input
: params.generation_prompt + input;
common_chat_input effective_input = input;
effective_input.prepend(params.generation_prompt);
//LOG_DBG("Parsing PEG input with format %s: %s\n", common_chat_format_name(params.format), effective_input.c_str());
//LOG_DBG("Parsing PEG input with format %s: %s\n", common_chat_format_name(params.format), effective_input.text.c_str());
common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_LENIENT;
if (params.debug) {
flags |= COMMON_PEG_PARSE_FLAG_DEBUG;
}
common_peg_parse_context ctx(effective_input, flags);
common_peg_parse_context ctx(std::move(effective_input.text), std::move(effective_input.tokens), flags);
auto result = parser.parse(ctx);
if (result.fail()) {
@@ -1507,8 +1562,8 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars
}
return msg;
}
LOG_WRN("%s: unparsed %s output: %s\n", __func__, common_chat_format_name(params.format), effective_input.substr(result.end).c_str());
LOG_DBG("%s: full %s output triggering error:\n=== BEGIN ===\n%s\n=== END ===\n", __func__, common_chat_format_name(params.format), effective_input.c_str());
LOG_WRN("%s: unparsed %s output: %s\n", __func__, common_chat_format_name(params.format), ctx.input.substr(result.end).c_str());
LOG_DBG("%s: full %s output triggering error:\n=== BEGIN ===\n%s\n=== END ===\n", __func__, common_chat_format_name(params.format), ctx.input.c_str());
throw std::runtime_error(std::string("The model produced output that does not match the expected ") + common_chat_format_name(params.format) + " format");
}
+29 -4
View File
@@ -282,6 +282,31 @@ struct common_chat_params {
common_chat_msg_delimiters message_delimiters;
};
struct common_chat_input {
std::string text;
std::vector<llama_token> tokens;
common_chat_input() = default;
// plain text, with no tokens
explicit common_chat_input(std::string text) : text(std::move(text)), tokens(this->text.size(), LLAMA_TOKEN_NULL) {}
size_t size() const { return text.size(); }
bool empty() const { return text.empty(); }
void append(const std::string & piece, llama_token token);
void append(const common_chat_input & chunk);
void prepend(const std::string & prefix);
void prepend(const common_chat_input & prefix);
void truncate(size_t pos);
common_chat_input substr(size_t pos, size_t n = std::string::npos) const;
};
common_chat_input common_chat_input_tokenize(const llama_vocab * vocab, const std::string & text);
// per-message parsing syntax
// should be derived from common_chat_params
struct common_chat_parser_params {
@@ -289,7 +314,7 @@ struct common_chat_parser_params {
common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_NONE; // TODO: refactor this to "bool parse_reasoning"
// Whether reasoning_content should be inlined in the content (e.g. for reasoning_format=deepseek in stream mode)
bool reasoning_in_content = false;
std::string generation_prompt;
common_chat_input generation_prompt;
bool parse_tool_calls = true;
bool is_continuation = false;
bool echo = false; // Include assistant prefilled msg in output
@@ -298,7 +323,7 @@ struct common_chat_parser_params {
common_chat_parser_params() = default;
common_chat_parser_params(const common_chat_params & chat_params) {
format = chat_params.format;
generation_prompt = chat_params.generation_prompt;
generation_prompt = common_chat_input(chat_params.generation_prompt);
}
};
@@ -337,8 +362,8 @@ std::string common_chat_format_example(const struct common_chat_templates *
const std::map<std::string, std::string> & chat_template_kwargs);
const char * common_chat_format_name(common_chat_format format);
common_chat_msg common_chat_parse(const std::string & input, bool is_partial, const common_chat_parser_params & params);
common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, const std::string & input, bool is_partial, const common_chat_parser_params & params);
common_chat_msg common_chat_parse(const common_chat_input & input, bool is_partial, const common_chat_parser_params & params);
common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, const common_chat_input & input, bool is_partial, const common_chat_parser_params & params);
// used by arg and server
const char * common_reasoning_format_name(common_reasoning_format format);
+8 -1
View File
@@ -2,6 +2,7 @@
#include "json-schema.h"
#include "json.h"
#include "llama.h"
#include <memory>
#include <set>
@@ -182,7 +183,8 @@ inline common_peg_parse_flags operator~(common_peg_parse_flags a) {
}
struct common_peg_parse_context {
std::string input;
std::string input; // [h, e, l, l, o, _, w, o, r, l, d]
std::vector<llama_token> tokens; // [id, -1, -1, -1, -1, id, -1, -1, -1, -1, -1]
common_peg_parse_flags flags;
common_peg_ast_arena ast;
@@ -194,6 +196,11 @@ struct common_peg_parse_context {
common_peg_parse_context(const std::string & input, common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_NONE)
: input(input), flags(flags), parse_depth(0) {}
common_peg_parse_context(std::string input, std::vector<llama_token> tokens, common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_NONE)
: input(std::move(input)), tokens(std::move(tokens)), flags(flags), parse_depth(0) {
GGML_ASSERT(this->tokens.empty() || this->tokens.size() == this->input.size());
}
bool is_lenient() const { return flags & COMMON_PEG_PARSE_FLAG_LENIENT; }
bool is_debug() const { return flags & COMMON_PEG_PARSE_FLAG_DEBUG; }
};
+2 -2
View File
@@ -1105,7 +1105,7 @@ struct make_peg_parser {
common_chat_msg parse(const std::string & msg, bool is_partial) const {
common_chat_parser_params parser_params(params_);
parser_params.debug = detailed_debug_;
return common_chat_peg_parse(arena_, msg, is_partial, parser_params);
return common_chat_peg_parse(arena_, common_chat_input(msg), is_partial, parser_params);
}
};
@@ -4643,7 +4643,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
bool got_out_of_range = false;
std::string error_msg;
try {
common_chat_peg_parse(arena, bad_input, /*is_partial=*/false, pp);
common_chat_peg_parse(arena, common_chat_input(bad_input), /*is_partial=*/false, pp);
} catch (const std::out_of_range & e) {
got_out_of_range = true;
error_msg = e.what();
+19 -21
View File
@@ -267,7 +267,7 @@ struct server_slot {
bool spec_is_replay = false;
std::mt19937 spec_synth_rng;
// TODO: move members that belong to the task (such as `generated_text`, `has_new_line`) to task_results_state
// TODO: move members that belong to the task (such as `generated`, `has_new_line`) to task_results_state
// see https://github.com/ggml-org/llama.cpp/pull/18283#issuecomment-3710175837
std::unique_ptr<const server_task> task;
std::unique_ptr<const server_task> task_prev; // used for debugging
@@ -285,7 +285,7 @@ struct server_slot {
size_t last_nl_pos = 0;
std::string generated_text;
common_chat_input generated;
std::string debug_generated_text;
llama_tokens generated_tokens;
size_t n_sent_text = 0; // number of sent text character (i.e. handle partial UTF-8 on streaming)
@@ -381,7 +381,7 @@ struct server_slot {
spec_is_replay = false;
last_nl_pos = 0;
generated_text = "";
generated = {};
has_new_line = false;
truncated = false;
stop = STOP_TYPE_NONE;
@@ -750,7 +750,7 @@ struct server_slot {
if (!only_metrics) {
res["prompt"] = ptask->tokens.detokenize(ctx_tgt, true);
res["generated"] = generated_text.empty() ? debug_generated_text : generated_text;
res["generated"] = generated.empty() ? debug_generated_text : generated.text;
}
}
@@ -1968,28 +1968,26 @@ private:
const std::string token_str = result.text_to_send;
slot.sampled = result.tok;
slot.generated_text += token_str;
slot.generated.append(token_str, result.tok);
if (slot.task->params.return_tokens) {
slot.generated_tokens.push_back(result.tok);
}
slot.has_next_token = true;
// check if there is incomplete UTF-8 character at the end
bool incomplete = validate_utf8(slot.generated_text) < slot.generated_text.size();
bool incomplete = validate_utf8(slot.generated.text) < slot.generated.text.size();
// search stop word and delete it
if (!incomplete) {
size_t pos = std::min(slot.n_sent_text, slot.generated_text.size());
size_t pos = std::min(slot.n_sent_text, slot.generated.text.size());
const std::string str_test = slot.generated_text.substr(pos);
const std::string str_test = slot.generated.text.substr(pos);
bool send_text = true;
size_t stop_pos = slot.find_stopping_strings(str_test, token_str.size(), true);
if (stop_pos != std::string::npos) {
slot.generated_text.erase(
slot.generated_text.begin() + pos + stop_pos,
slot.generated_text.end());
pos = std::min(slot.n_sent_text, slot.generated_text.size());
slot.generated.truncate(pos + stop_pos);
pos = std::min(slot.n_sent_text, slot.generated.text.size());
} else if (slot.has_next_token && !llama_vocab_is_eog(vocab, result.tok) ) {
stop_pos = slot.find_stopping_strings(str_test, token_str.size(), false);
send_text = stop_pos == std::string::npos;
@@ -1998,7 +1996,7 @@ private:
// check if there is any token to predict
if (send_text) {
// no send the stop word in the response
result.text_to_send = slot.generated_text.substr(pos, std::string::npos);
result.text_to_send = slot.generated.text.substr(pos, std::string::npos);
slot.n_sent_text += result.text_to_send.size();
// add the token to slot queue and cache
} else {
@@ -2042,17 +2040,17 @@ private:
size_t pos = slot.last_nl_pos;
int n_indent = 0;
while (pos < slot.generated_text.size() && (slot.generated_text[pos] == ' ' || slot.generated_text[pos] == '\t')) {
while (pos < slot.generated.text.size() && (slot.generated.text[pos] == ' ' || slot.generated.text[pos] == '\t')) {
n_indent++;
pos++;
}
if (pos < slot.generated_text.size() && n_indent < slot.task->params.n_indent) {
if (pos < slot.generated.text.size() && n_indent < slot.task->params.n_indent) {
slot.stop = STOP_TYPE_LIMIT;
slot.has_next_token = false;
// cut the last line
slot.generated_text.erase(pos, std::string::npos);
slot.generated.truncate(pos);
SLT_DBG(slot, "stopped by indentation limit, n_gen = %d, n_indent = %d\n", (int) slot.stats.n_gen, n_indent);
}
@@ -2060,7 +2058,7 @@ private:
// find the next new line
{
const size_t pos = slot.generated_text.find('\n', slot.last_nl_pos);
const size_t pos = slot.generated.text.find('\n', slot.last_nl_pos);
if (pos != std::string::npos) {
slot.last_nl_pos = pos + 1;
@@ -2192,7 +2190,7 @@ private:
if (is_begin) {
res->is_begin = true;
} else {
res->content = tkn.text_to_send;
res->content = slot.generated.substr(slot.n_sent_text - tkn.text_to_send.size(), tkn.text_to_send.size());
res->tokens.assign(1, tkn.tok);
}
@@ -2229,15 +2227,15 @@ private:
// keep copy of last generated text for debugging purposes
if (slots_debug) {
slot.debug_generated_text = slot.generated_text;
slot.debug_generated_text = slot.generated.text;
}
// in stream mode, content and tokens are already in last partial chunk
if (slot.task->params.stream) {
res->content = "";
res->content = {};
res->tokens = llama_tokens{};
} else {
res->content = std::move(slot.generated_text);
res->content = std::move(slot.generated);
res->tokens = std::move(slot.generated_tokens);
}
res->stats = slot.stats;
+7 -1
View File
@@ -311,8 +311,14 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
->set_desc("Generation prompt appended to the chat template output")
->set_handler([&](field_eval_context & ctx, const json & data) {
std::string s = data.at("generation_prompt").get<std::string>();
ctx.params.chat_parser_params.generation_prompt = s;
ctx.params.sampling.generation_prompt = s;
if (ctx.vocab == nullptr) {
ctx.params.chat_parser_params.generation_prompt = common_chat_input(s);
return;
}
ctx.params.chat_parser_params.generation_prompt = common_chat_input_tokenize(ctx.vocab, s);
}));
add((new field_bool("parse_tool_calls", params.chat_parser_params.parse_tool_calls))
+15 -15
View File
@@ -76,7 +76,7 @@ json task_params::to_json(bool only_metrics) const {
{"chat_format", common_chat_format_name(chat_parser_params.format)},
{"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)},
{"reasoning_in_content", chat_parser_params.reasoning_in_content},
{"generation_prompt", chat_parser_params.generation_prompt},
{"generation_prompt", chat_parser_params.generation_prompt.text},
{"samplers", samplers},
{"speculative.types", common_speculative_type_name_str(speculative.types)},
{"timings_per_token", timings_per_token},
@@ -135,7 +135,7 @@ json task_params::to_json(bool only_metrics) const {
{"chat_format", common_chat_format_name(chat_parser_params.format)},
{"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)},
{"reasoning_in_content", chat_parser_params.reasoning_in_content},
{"generation_prompt", chat_parser_params.generation_prompt},
{"generation_prompt", chat_parser_params.generation_prompt.text},
{"samplers", samplers},
{"speculative.types", common_speculative_type_name_str(speculative.types)},
{"timings_per_token", timings_per_token},
@@ -155,20 +155,20 @@ task_result_state::task_result_state(const common_chat_parser_params & chat_pars
, oai_resp_message_id("msg_" + random_string()) {
if (chat_parser_params.is_continuation && !chat_parser_params.echo) {
// initialize chat_msg to avoid emitting a delta containing the assistant prefill
chat_msg = common_chat_parse("", true, chat_parser_params);
chat_msg = common_chat_parse(generated_input, true, chat_parser_params);
}
}
common_chat_msg task_result_state::update_chat_msg(
const std::string & text_added,
const common_chat_input & added,
bool is_partial,
std::vector<common_chat_msg_diff> & diffs,
bool filter_tool_calls) {
generated_text += text_added;
generated_input.append(added);
auto msg_prv_copy = chat_msg;
//SRV_DBG("Parsing chat message: %s\n", generated_text.c_str());
//SRV_DBG("Parsing chat message: %s\n", generated_input.text.c_str());
auto new_msg = common_chat_parse(
generated_text,
generated_input,
is_partial,
chat_parser_params);
if (!new_msg.empty()) {
@@ -340,7 +340,7 @@ json server_task_result_cmpl_final::to_json() {
json server_task_result_cmpl_final::to_json_non_oaicompat() {
json res = json {
{"index", index},
{"content", content},
{"content", content.text},
{"tokens", tokens},
{"id_slot", id_slot},
{"stop", true},
@@ -386,7 +386,7 @@ json server_task_result_cmpl_final::to_json_oaicompat() {
json res = json {
{"choices", json::array({
json{
{"text", content},
{"text", content.text},
{"index", index},
{"logprobs", logprobs},
{"finish_reason", finish_reason},
@@ -418,7 +418,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat() {
msg = oaicompat_msg;
} else {
msg.role = "assistant";
msg.content = content;
msg.content = content.text;
}
if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {
finish_reason = msg.tool_calls.empty() ? "stop" : "tool_calls";
@@ -531,7 +531,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp() {
msg = oaicompat_msg;
} else {
msg.role = "assistant";
msg.content = content;
msg.content = content.text;
}
std::vector<json> output;
@@ -741,7 +741,7 @@ json server_task_result_cmpl_final::to_json_anthropic() {
msg = oaicompat_msg;
} else {
msg.role = "assistant";
msg.content = content;
msg.content = content.text;
}
// thinking block comes first (Anthropic extended thinking format)
@@ -1050,7 +1050,7 @@ json server_task_result_cmpl_partial::to_json_non_oaicompat() {
// non-OAI-compat JSON
json res = json {
{"index", index},
{"content", content},
{"content", content.text},
{"tokens", tokens},
{"stop", false},
{"id_slot", id_slot},
@@ -1081,7 +1081,7 @@ json server_task_result_cmpl_partial::to_json_oaicompat() {
json res = json {
{"choices", json::array({
json{
{"text", content},
{"text", content.text},
{"index", index},
{"logprobs", logprobs},
{"finish_reason", nullptr},
@@ -1316,7 +1316,7 @@ json server_task_result_cmpl_partial::to_json_oaicompat_resp() {
json server_task_result_cmpl_partial::to_json_oaicompat_asr() {
json event = json {
{"type", "transcript.text.delta"},
{"delta", content},
{"delta", content.text},
};
return event;
}
+5 -5
View File
@@ -108,7 +108,7 @@ struct task_result_state {
std::vector<common_chat_msg_diff> diffs;
common_chat_parser_params chat_parser_params;
common_chat_msg chat_msg;
std::string generated_text; // append new chunks of generated text here
common_chat_input generated_input; // append new chunks of generated text here
std::vector<std::string> generated_tool_call_ids;
std::unordered_set<size_t> sent_tool_call_names;
@@ -128,7 +128,7 @@ struct task_result_state {
// parse partial tool calls and update the internal state
common_chat_msg update_chat_msg(
const std::string & text_added,
const common_chat_input & added,
bool is_partial,
std::vector<common_chat_msg_diff> & diffs,
bool filter_tool_calls = false);
@@ -350,7 +350,7 @@ struct completion_token_output {
};
struct server_task_result_cmpl_final : server_task_result {
std::string content;
common_chat_input content;
llama_tokens tokens;
bool stream;
@@ -425,8 +425,8 @@ struct server_task_result_cmpl_final : server_task_result {
};
struct server_task_result_cmpl_partial : server_task_result {
std::string content;
llama_tokens tokens;
common_chat_input content;
llama_tokens tokens;
int32_t n_decoded;
int32_t n_prompt_tokens;