server : accumulate generated text and tokens as parse input (#29876)

* server : collect text and token input

* common : add a tokenize helper that aligns tokens with bytes

* common : simplify tokenize logic
This commit is contained in:
Aldehir Rojas authored and GitHub committed 2026-10-07 11:49:17 -05:00
1 parent 448147d42a
commit 18b5f8b186
8 files changed
+149 -58

No files matched your search

+19 -21
View File
@@ -267,7 +267,7 @@ struct server_slot {
bool spec_is_replay = false;
std::mt19937 spec_synth_rng;
// TODO: move members that belong to the task (such as `generated_text`, `has_new_line`) to task_results_state
// TODO: move members that belong to the task (such as `generated`, `has_new_line`) to task_results_state
// see https://github.com/ggml-org/llama.cpp/pull/18283#issuecomment-3710175837
std::unique_ptr<const server_task> task;
std::unique_ptr<const server_task> task_prev; // used for debugging
@@ -285,7 +285,7 @@ struct server_slot {
size_t last_nl_pos = 0;
std::string generated_text;
common_chat_input generated;
std::string debug_generated_text;
llama_tokens generated_tokens;
size_t n_sent_text = 0; // number of sent text character (i.e. handle partial UTF-8 on streaming)
@@ -381,7 +381,7 @@ struct server_slot {
spec_is_replay = false;
last_nl_pos = 0;
generated_text = "";
generated = {};
has_new_line = false;
truncated = false;
stop = STOP_TYPE_NONE;
@@ -750,7 +750,7 @@ struct server_slot {
if (!only_metrics) {
res["prompt"] = ptask->tokens.detokenize(ctx_tgt, true);
res["generated"] = generated_text.empty() ? debug_generated_text : generated_text;
res["generated"] = generated.empty() ? debug_generated_text : generated.text;
}
}
@@ -1968,28 +1968,26 @@ private:
const std::string token_str = result.text_to_send;
slot.sampled = result.tok;
slot.generated_text += token_str;
slot.generated.append(token_str, result.tok);
if (slot.task->params.return_tokens) {
slot.generated_tokens.push_back(result.tok);
}
slot.has_next_token = true;
// check if there is incomplete UTF-8 character at the end
bool incomplete = validate_utf8(slot.generated_text) < slot.generated_text.size();
bool incomplete = validate_utf8(slot.generated.text) < slot.generated.text.size();
// search stop word and delete it
if (!incomplete) {
size_t pos = std::min(slot.n_sent_text, slot.generated_text.size());
size_t pos = std::min(slot.n_sent_text, slot.generated.text.size());
const std::string str_test = slot.generated_text.substr(pos);
const std::string str_test = slot.generated.text.substr(pos);
bool send_text = true;
size_t stop_pos = slot.find_stopping_strings(str_test, token_str.size(), true);
if (stop_pos != std::string::npos) {
slot.generated_text.erase(
slot.generated_text.begin() + pos + stop_pos,
slot.generated_text.end());
pos = std::min(slot.n_sent_text, slot.generated_text.size());
slot.generated.truncate(pos + stop_pos);
pos = std::min(slot.n_sent_text, slot.generated.text.size());
} else if (slot.has_next_token && !llama_vocab_is_eog(vocab, result.tok) ) {
stop_pos = slot.find_stopping_strings(str_test, token_str.size(), false);
send_text = stop_pos == std::string::npos;
@@ -1998,7 +1996,7 @@ private:
// check if there is any token to predict
if (send_text) {
// no send the stop word in the response
result.text_to_send = slot.generated_text.substr(pos, std::string::npos);
result.text_to_send = slot.generated.text.substr(pos, std::string::npos);
slot.n_sent_text += result.text_to_send.size();
// add the token to slot queue and cache
} else {
@@ -2042,17 +2040,17 @@ private:
size_t pos = slot.last_nl_pos;
int n_indent = 0;
while (pos < slot.generated_text.size() && (slot.generated_text[pos] == ' ' || slot.generated_text[pos] == '\t')) {
while (pos < slot.generated.text.size() && (slot.generated.text[pos] == ' ' || slot.generated.text[pos] == '\t')) {
n_indent++;
pos++;
}
if (pos < slot.generated_text.size() && n_indent < slot.task->params.n_indent) {
if (pos < slot.generated.text.size() && n_indent < slot.task->params.n_indent) {
slot.stop = STOP_TYPE_LIMIT;
slot.has_next_token = false;
// cut the last line
slot.generated_text.erase(pos, std::string::npos);
slot.generated.truncate(pos);
SLT_DBG(slot, "stopped by indentation limit, n_gen = %d, n_indent = %d\n", (int) slot.stats.n_gen, n_indent);
}
@@ -2060,7 +2058,7 @@ private:
// find the next new line
{
const size_t pos = slot.generated_text.find('\n', slot.last_nl_pos);
const size_t pos = slot.generated.text.find('\n', slot.last_nl_pos);
if (pos != std::string::npos) {
slot.last_nl_pos = pos + 1;
@@ -2192,7 +2190,7 @@ private:
if (is_begin) {
res->is_begin = true;
} else {
res->content = tkn.text_to_send;
res->content = slot.generated.substr(slot.n_sent_text - tkn.text_to_send.size(), tkn.text_to_send.size());
res->tokens.assign(1, tkn.tok);
}
@@ -2229,15 +2227,15 @@ private:
// keep copy of last generated text for debugging purposes
if (slots_debug) {
slot.debug_generated_text = slot.generated_text;
slot.debug_generated_text = slot.generated.text;
}
// in stream mode, content and tokens are already in last partial chunk
if (slot.task->params.stream) {
res->content = "";
res->content = {};
res->tokens = llama_tokens{};
} else {
res->content = std::move(slot.generated_text);
res->content = std::move(slot.generated);
res->tokens = std::move(slot.generated_tokens);
}
res->stats = slot.stats;