server : accumulate generated text and tokens as parse input (#29876)
* server : collect text and token input * common : add a tokenize helper that aligns tokens with bytes * common : simplify tokenize logic
This commit is contained in:
1 parent
448147d42a
commit
18b5f8b186
8 files changed
+149
-58
No files matched your search
+64
-9
@@ -1452,14 +1452,70 @@ common_chat_params common_chat_templates_apply(const struct common_chat_template
|
||||
common_chat_templates_apply_legacy(tmpls, inputs);
|
||||
}
|
||||
|
||||
common_chat_msg common_chat_parse(const std::string & input,
|
||||
void common_chat_input::append(const std::string & piece, llama_token token) {
|
||||
if (piece.empty()) {
|
||||
return;
|
||||
}
|
||||
tokens.push_back(token);
|
||||
tokens.resize(tokens.size() + piece.size() - 1, LLAMA_TOKEN_NULL);
|
||||
text += piece;
|
||||
}
|
||||
|
||||
void common_chat_input::append(const common_chat_input & chunk) {
|
||||
tokens.insert(tokens.end(), chunk.tokens.begin(), chunk.tokens.end());
|
||||
text += chunk.text;
|
||||
}
|
||||
|
||||
void common_chat_input::truncate(size_t pos) {
|
||||
if (pos < text.size()) {
|
||||
text.erase(pos);
|
||||
tokens.resize(pos);
|
||||
}
|
||||
}
|
||||
|
||||
common_chat_input common_chat_input::substr(size_t pos, size_t n) const {
|
||||
common_chat_input out;
|
||||
out.text = text.substr(pos, n);
|
||||
out.tokens.assign(tokens.begin() + pos, tokens.begin() + pos + out.size());
|
||||
return out;
|
||||
}
|
||||
|
||||
void common_chat_input::prepend(const std::string & prefix) {
|
||||
tokens.insert(tokens.begin(), prefix.size(), LLAMA_TOKEN_NULL);
|
||||
text = prefix + text;
|
||||
}
|
||||
|
||||
void common_chat_input::prepend(const common_chat_input & prefix) {
|
||||
tokens.insert(tokens.begin(), prefix.tokens.begin(), prefix.tokens.end());
|
||||
text = prefix.text + text;
|
||||
}
|
||||
|
||||
common_chat_input common_chat_input_tokenize(const llama_vocab * vocab, const std::string & text) {
|
||||
common_chat_input input;
|
||||
auto tokens = common_tokenize(vocab, text, false, true);
|
||||
for (size_t i = 0; i < tokens.size(); i++) {
|
||||
std::string piece = common_token_to_piece(vocab, tokens[i], true);
|
||||
if (i == 0 && std::isspace(piece[0]) && !std::isspace(text[0])) {
|
||||
// Some tokenizers will add a space before the first special token, need to exclude
|
||||
continue;
|
||||
}
|
||||
input.append(piece, tokens[i]);
|
||||
}
|
||||
if (input.text != text) {
|
||||
// the pieces do not give back the same text, keep the text without tokens
|
||||
return common_chat_input(text);
|
||||
}
|
||||
return input;
|
||||
}
|
||||
|
||||
common_chat_msg common_chat_parse(const common_chat_input & input,
|
||||
bool is_partial,
|
||||
const common_chat_parser_params & params) {
|
||||
return common_chat_peg_parse(params.parser, input, is_partial, params);
|
||||
}
|
||||
|
||||
common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser,
|
||||
const std::string & input,
|
||||
const common_chat_input & input,
|
||||
bool is_partial,
|
||||
const common_chat_parser_params & params) {
|
||||
const common_peg_arena & parser = src_parser.empty() ?
|
||||
@@ -1470,18 +1526,17 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars
|
||||
LOG_DBG("No parser definition detected, assuming pure content parser.");
|
||||
}
|
||||
|
||||
const std::string effective_input = params.generation_prompt.empty()
|
||||
? input
|
||||
: params.generation_prompt + input;
|
||||
common_chat_input effective_input = input;
|
||||
effective_input.prepend(params.generation_prompt);
|
||||
|
||||
//LOG_DBG("Parsing PEG input with format %s: %s\n", common_chat_format_name(params.format), effective_input.c_str());
|
||||
//LOG_DBG("Parsing PEG input with format %s: %s\n", common_chat_format_name(params.format), effective_input.text.c_str());
|
||||
|
||||
common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_LENIENT;
|
||||
if (params.debug) {
|
||||
flags |= COMMON_PEG_PARSE_FLAG_DEBUG;
|
||||
}
|
||||
|
||||
common_peg_parse_context ctx(effective_input, flags);
|
||||
common_peg_parse_context ctx(std::move(effective_input.text), std::move(effective_input.tokens), flags);
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
if (result.fail()) {
|
||||
@@ -1507,8 +1562,8 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars
|
||||
}
|
||||
return msg;
|
||||
}
|
||||
LOG_WRN("%s: unparsed %s output: %s\n", __func__, common_chat_format_name(params.format), effective_input.substr(result.end).c_str());
|
||||
LOG_DBG("%s: full %s output triggering error:\n=== BEGIN ===\n%s\n=== END ===\n", __func__, common_chat_format_name(params.format), effective_input.c_str());
|
||||
LOG_WRN("%s: unparsed %s output: %s\n", __func__, common_chat_format_name(params.format), ctx.input.substr(result.end).c_str());
|
||||
LOG_DBG("%s: full %s output triggering error:\n=== BEGIN ===\n%s\n=== END ===\n", __func__, common_chat_format_name(params.format), ctx.input.c_str());
|
||||
throw std::runtime_error(std::string("The model produced output that does not match the expected ") + common_chat_format_name(params.format) + " format");
|
||||
}
|
||||
|
||||
|
||||
+29
-4
@@ -282,6 +282,31 @@ struct common_chat_params {
|
||||
common_chat_msg_delimiters message_delimiters;
|
||||
};
|
||||
|
||||
struct common_chat_input {
|
||||
std::string text;
|
||||
std::vector<llama_token> tokens;
|
||||
|
||||
common_chat_input() = default;
|
||||
|
||||
// plain text, with no tokens
|
||||
explicit common_chat_input(std::string text) : text(std::move(text)), tokens(this->text.size(), LLAMA_TOKEN_NULL) {}
|
||||
|
||||
size_t size() const { return text.size(); }
|
||||
bool empty() const { return text.empty(); }
|
||||
|
||||
void append(const std::string & piece, llama_token token);
|
||||
void append(const common_chat_input & chunk);
|
||||
|
||||
void prepend(const std::string & prefix);
|
||||
void prepend(const common_chat_input & prefix);
|
||||
|
||||
void truncate(size_t pos);
|
||||
|
||||
common_chat_input substr(size_t pos, size_t n = std::string::npos) const;
|
||||
};
|
||||
|
||||
common_chat_input common_chat_input_tokenize(const llama_vocab * vocab, const std::string & text);
|
||||
|
||||
// per-message parsing syntax
|
||||
// should be derived from common_chat_params
|
||||
struct common_chat_parser_params {
|
||||
@@ -289,7 +314,7 @@ struct common_chat_parser_params {
|
||||
common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_NONE; // TODO: refactor this to "bool parse_reasoning"
|
||||
// Whether reasoning_content should be inlined in the content (e.g. for reasoning_format=deepseek in stream mode)
|
||||
bool reasoning_in_content = false;
|
||||
std::string generation_prompt;
|
||||
common_chat_input generation_prompt;
|
||||
bool parse_tool_calls = true;
|
||||
bool is_continuation = false;
|
||||
bool echo = false; // Include assistant prefilled msg in output
|
||||
@@ -298,7 +323,7 @@ struct common_chat_parser_params {
|
||||
common_chat_parser_params() = default;
|
||||
common_chat_parser_params(const common_chat_params & chat_params) {
|
||||
format = chat_params.format;
|
||||
generation_prompt = chat_params.generation_prompt;
|
||||
generation_prompt = common_chat_input(chat_params.generation_prompt);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -337,8 +362,8 @@ std::string common_chat_format_example(const struct common_chat_templates *
|
||||
const std::map<std::string, std::string> & chat_template_kwargs);
|
||||
|
||||
const char * common_chat_format_name(common_chat_format format);
|
||||
common_chat_msg common_chat_parse(const std::string & input, bool is_partial, const common_chat_parser_params & params);
|
||||
common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, const std::string & input, bool is_partial, const common_chat_parser_params & params);
|
||||
common_chat_msg common_chat_parse(const common_chat_input & input, bool is_partial, const common_chat_parser_params & params);
|
||||
common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, const common_chat_input & input, bool is_partial, const common_chat_parser_params & params);
|
||||
|
||||
// used by arg and server
|
||||
const char * common_reasoning_format_name(common_reasoning_format format);
|
||||
|
||||
+8
-1
@@ -2,6 +2,7 @@
|
||||
|
||||
#include "json-schema.h"
|
||||
#include "json.h"
|
||||
#include "llama.h"
|
||||
|
||||
#include <memory>
|
||||
#include <set>
|
||||
@@ -182,7 +183,8 @@ inline common_peg_parse_flags operator~(common_peg_parse_flags a) {
|
||||
}
|
||||
|
||||
struct common_peg_parse_context {
|
||||
std::string input;
|
||||
std::string input; // [h, e, l, l, o, _, w, o, r, l, d]
|
||||
std::vector<llama_token> tokens; // [id, -1, -1, -1, -1, id, -1, -1, -1, -1, -1]
|
||||
common_peg_parse_flags flags;
|
||||
common_peg_ast_arena ast;
|
||||
|
||||
@@ -194,6 +196,11 @@ struct common_peg_parse_context {
|
||||
common_peg_parse_context(const std::string & input, common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_NONE)
|
||||
: input(input), flags(flags), parse_depth(0) {}
|
||||
|
||||
common_peg_parse_context(std::string input, std::vector<llama_token> tokens, common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_NONE)
|
||||
: input(std::move(input)), tokens(std::move(tokens)), flags(flags), parse_depth(0) {
|
||||
GGML_ASSERT(this->tokens.empty() || this->tokens.size() == this->input.size());
|
||||
}
|
||||
|
||||
bool is_lenient() const { return flags & COMMON_PEG_PARSE_FLAG_LENIENT; }
|
||||
bool is_debug() const { return flags & COMMON_PEG_PARSE_FLAG_DEBUG; }
|
||||
};
|
||||
|
||||
+2
-2
@@ -1105,7 +1105,7 @@ struct make_peg_parser {
|
||||
common_chat_msg parse(const std::string & msg, bool is_partial) const {
|
||||
common_chat_parser_params parser_params(params_);
|
||||
parser_params.debug = detailed_debug_;
|
||||
return common_chat_peg_parse(arena_, msg, is_partial, parser_params);
|
||||
return common_chat_peg_parse(arena_, common_chat_input(msg), is_partial, parser_params);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -4643,7 +4643,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
|
||||
bool got_out_of_range = false;
|
||||
std::string error_msg;
|
||||
try {
|
||||
common_chat_peg_parse(arena, bad_input, /*is_partial=*/false, pp);
|
||||
common_chat_peg_parse(arena, common_chat_input(bad_input), /*is_partial=*/false, pp);
|
||||
} catch (const std::out_of_range & e) {
|
||||
got_out_of_range = true;
|
||||
error_msg = e.what();
|
||||
|
||||
@@ -267,7 +267,7 @@ struct server_slot {
|
||||
bool spec_is_replay = false;
|
||||
std::mt19937 spec_synth_rng;
|
||||
|
||||
// TODO: move members that belong to the task (such as `generated_text`, `has_new_line`) to task_results_state
|
||||
// TODO: move members that belong to the task (such as `generated`, `has_new_line`) to task_results_state
|
||||
// see https://github.com/ggml-org/llama.cpp/pull/18283#issuecomment-3710175837
|
||||
std::unique_ptr<const server_task> task;
|
||||
std::unique_ptr<const server_task> task_prev; // used for debugging
|
||||
@@ -285,7 +285,7 @@ struct server_slot {
|
||||
|
||||
size_t last_nl_pos = 0;
|
||||
|
||||
std::string generated_text;
|
||||
common_chat_input generated;
|
||||
std::string debug_generated_text;
|
||||
llama_tokens generated_tokens;
|
||||
size_t n_sent_text = 0; // number of sent text character (i.e. handle partial UTF-8 on streaming)
|
||||
@@ -381,7 +381,7 @@ struct server_slot {
|
||||
spec_is_replay = false;
|
||||
|
||||
last_nl_pos = 0;
|
||||
generated_text = "";
|
||||
generated = {};
|
||||
has_new_line = false;
|
||||
truncated = false;
|
||||
stop = STOP_TYPE_NONE;
|
||||
@@ -750,7 +750,7 @@ struct server_slot {
|
||||
|
||||
if (!only_metrics) {
|
||||
res["prompt"] = ptask->tokens.detokenize(ctx_tgt, true);
|
||||
res["generated"] = generated_text.empty() ? debug_generated_text : generated_text;
|
||||
res["generated"] = generated.empty() ? debug_generated_text : generated.text;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1968,28 +1968,26 @@ private:
|
||||
const std::string token_str = result.text_to_send;
|
||||
slot.sampled = result.tok;
|
||||
|
||||
slot.generated_text += token_str;
|
||||
slot.generated.append(token_str, result.tok);
|
||||
if (slot.task->params.return_tokens) {
|
||||
slot.generated_tokens.push_back(result.tok);
|
||||
}
|
||||
slot.has_next_token = true;
|
||||
|
||||
// check if there is incomplete UTF-8 character at the end
|
||||
bool incomplete = validate_utf8(slot.generated_text) < slot.generated_text.size();
|
||||
bool incomplete = validate_utf8(slot.generated.text) < slot.generated.text.size();
|
||||
|
||||
// search stop word and delete it
|
||||
if (!incomplete) {
|
||||
size_t pos = std::min(slot.n_sent_text, slot.generated_text.size());
|
||||
size_t pos = std::min(slot.n_sent_text, slot.generated.text.size());
|
||||
|
||||
const std::string str_test = slot.generated_text.substr(pos);
|
||||
const std::string str_test = slot.generated.text.substr(pos);
|
||||
bool send_text = true;
|
||||
|
||||
size_t stop_pos = slot.find_stopping_strings(str_test, token_str.size(), true);
|
||||
if (stop_pos != std::string::npos) {
|
||||
slot.generated_text.erase(
|
||||
slot.generated_text.begin() + pos + stop_pos,
|
||||
slot.generated_text.end());
|
||||
pos = std::min(slot.n_sent_text, slot.generated_text.size());
|
||||
slot.generated.truncate(pos + stop_pos);
|
||||
pos = std::min(slot.n_sent_text, slot.generated.text.size());
|
||||
} else if (slot.has_next_token && !llama_vocab_is_eog(vocab, result.tok) ) {
|
||||
stop_pos = slot.find_stopping_strings(str_test, token_str.size(), false);
|
||||
send_text = stop_pos == std::string::npos;
|
||||
@@ -1998,7 +1996,7 @@ private:
|
||||
// check if there is any token to predict
|
||||
if (send_text) {
|
||||
// no send the stop word in the response
|
||||
result.text_to_send = slot.generated_text.substr(pos, std::string::npos);
|
||||
result.text_to_send = slot.generated.text.substr(pos, std::string::npos);
|
||||
slot.n_sent_text += result.text_to_send.size();
|
||||
// add the token to slot queue and cache
|
||||
} else {
|
||||
@@ -2042,17 +2040,17 @@ private:
|
||||
size_t pos = slot.last_nl_pos;
|
||||
|
||||
int n_indent = 0;
|
||||
while (pos < slot.generated_text.size() && (slot.generated_text[pos] == ' ' || slot.generated_text[pos] == '\t')) {
|
||||
while (pos < slot.generated.text.size() && (slot.generated.text[pos] == ' ' || slot.generated.text[pos] == '\t')) {
|
||||
n_indent++;
|
||||
pos++;
|
||||
}
|
||||
|
||||
if (pos < slot.generated_text.size() && n_indent < slot.task->params.n_indent) {
|
||||
if (pos < slot.generated.text.size() && n_indent < slot.task->params.n_indent) {
|
||||
slot.stop = STOP_TYPE_LIMIT;
|
||||
slot.has_next_token = false;
|
||||
|
||||
// cut the last line
|
||||
slot.generated_text.erase(pos, std::string::npos);
|
||||
slot.generated.truncate(pos);
|
||||
|
||||
SLT_DBG(slot, "stopped by indentation limit, n_gen = %d, n_indent = %d\n", (int) slot.stats.n_gen, n_indent);
|
||||
}
|
||||
@@ -2060,7 +2058,7 @@ private:
|
||||
|
||||
// find the next new line
|
||||
{
|
||||
const size_t pos = slot.generated_text.find('\n', slot.last_nl_pos);
|
||||
const size_t pos = slot.generated.text.find('\n', slot.last_nl_pos);
|
||||
|
||||
if (pos != std::string::npos) {
|
||||
slot.last_nl_pos = pos + 1;
|
||||
@@ -2192,7 +2190,7 @@ private:
|
||||
if (is_begin) {
|
||||
res->is_begin = true;
|
||||
} else {
|
||||
res->content = tkn.text_to_send;
|
||||
res->content = slot.generated.substr(slot.n_sent_text - tkn.text_to_send.size(), tkn.text_to_send.size());
|
||||
res->tokens.assign(1, tkn.tok);
|
||||
}
|
||||
|
||||
@@ -2229,15 +2227,15 @@ private:
|
||||
|
||||
// keep copy of last generated text for debugging purposes
|
||||
if (slots_debug) {
|
||||
slot.debug_generated_text = slot.generated_text;
|
||||
slot.debug_generated_text = slot.generated.text;
|
||||
}
|
||||
|
||||
// in stream mode, content and tokens are already in last partial chunk
|
||||
if (slot.task->params.stream) {
|
||||
res->content = "";
|
||||
res->content = {};
|
||||
res->tokens = llama_tokens{};
|
||||
} else {
|
||||
res->content = std::move(slot.generated_text);
|
||||
res->content = std::move(slot.generated);
|
||||
res->tokens = std::move(slot.generated_tokens);
|
||||
}
|
||||
res->stats = slot.stats;
|
||||
|
||||
@@ -311,8 +311,14 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
|
||||
->set_desc("Generation prompt appended to the chat template output")
|
||||
->set_handler([&](field_eval_context & ctx, const json & data) {
|
||||
std::string s = data.at("generation_prompt").get<std::string>();
|
||||
ctx.params.chat_parser_params.generation_prompt = s;
|
||||
ctx.params.sampling.generation_prompt = s;
|
||||
|
||||
if (ctx.vocab == nullptr) {
|
||||
ctx.params.chat_parser_params.generation_prompt = common_chat_input(s);
|
||||
return;
|
||||
}
|
||||
|
||||
ctx.params.chat_parser_params.generation_prompt = common_chat_input_tokenize(ctx.vocab, s);
|
||||
}));
|
||||
|
||||
add((new field_bool("parse_tool_calls", params.chat_parser_params.parse_tool_calls))
|
||||
|
||||
@@ -76,7 +76,7 @@ json task_params::to_json(bool only_metrics) const {
|
||||
{"chat_format", common_chat_format_name(chat_parser_params.format)},
|
||||
{"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)},
|
||||
{"reasoning_in_content", chat_parser_params.reasoning_in_content},
|
||||
{"generation_prompt", chat_parser_params.generation_prompt},
|
||||
{"generation_prompt", chat_parser_params.generation_prompt.text},
|
||||
{"samplers", samplers},
|
||||
{"speculative.types", common_speculative_type_name_str(speculative.types)},
|
||||
{"timings_per_token", timings_per_token},
|
||||
@@ -135,7 +135,7 @@ json task_params::to_json(bool only_metrics) const {
|
||||
{"chat_format", common_chat_format_name(chat_parser_params.format)},
|
||||
{"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)},
|
||||
{"reasoning_in_content", chat_parser_params.reasoning_in_content},
|
||||
{"generation_prompt", chat_parser_params.generation_prompt},
|
||||
{"generation_prompt", chat_parser_params.generation_prompt.text},
|
||||
{"samplers", samplers},
|
||||
{"speculative.types", common_speculative_type_name_str(speculative.types)},
|
||||
{"timings_per_token", timings_per_token},
|
||||
@@ -155,20 +155,20 @@ task_result_state::task_result_state(const common_chat_parser_params & chat_pars
|
||||
, oai_resp_message_id("msg_" + random_string()) {
|
||||
if (chat_parser_params.is_continuation && !chat_parser_params.echo) {
|
||||
// initialize chat_msg to avoid emitting a delta containing the assistant prefill
|
||||
chat_msg = common_chat_parse("", true, chat_parser_params);
|
||||
chat_msg = common_chat_parse(generated_input, true, chat_parser_params);
|
||||
}
|
||||
}
|
||||
|
||||
common_chat_msg task_result_state::update_chat_msg(
|
||||
const std::string & text_added,
|
||||
const common_chat_input & added,
|
||||
bool is_partial,
|
||||
std::vector<common_chat_msg_diff> & diffs,
|
||||
bool filter_tool_calls) {
|
||||
generated_text += text_added;
|
||||
generated_input.append(added);
|
||||
auto msg_prv_copy = chat_msg;
|
||||
//SRV_DBG("Parsing chat message: %s\n", generated_text.c_str());
|
||||
//SRV_DBG("Parsing chat message: %s\n", generated_input.text.c_str());
|
||||
auto new_msg = common_chat_parse(
|
||||
generated_text,
|
||||
generated_input,
|
||||
is_partial,
|
||||
chat_parser_params);
|
||||
if (!new_msg.empty()) {
|
||||
@@ -340,7 +340,7 @@ json server_task_result_cmpl_final::to_json() {
|
||||
json server_task_result_cmpl_final::to_json_non_oaicompat() {
|
||||
json res = json {
|
||||
{"index", index},
|
||||
{"content", content},
|
||||
{"content", content.text},
|
||||
{"tokens", tokens},
|
||||
{"id_slot", id_slot},
|
||||
{"stop", true},
|
||||
@@ -386,7 +386,7 @@ json server_task_result_cmpl_final::to_json_oaicompat() {
|
||||
json res = json {
|
||||
{"choices", json::array({
|
||||
json{
|
||||
{"text", content},
|
||||
{"text", content.text},
|
||||
{"index", index},
|
||||
{"logprobs", logprobs},
|
||||
{"finish_reason", finish_reason},
|
||||
@@ -418,7 +418,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat() {
|
||||
msg = oaicompat_msg;
|
||||
} else {
|
||||
msg.role = "assistant";
|
||||
msg.content = content;
|
||||
msg.content = content.text;
|
||||
}
|
||||
if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {
|
||||
finish_reason = msg.tool_calls.empty() ? "stop" : "tool_calls";
|
||||
@@ -531,7 +531,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp() {
|
||||
msg = oaicompat_msg;
|
||||
} else {
|
||||
msg.role = "assistant";
|
||||
msg.content = content;
|
||||
msg.content = content.text;
|
||||
}
|
||||
|
||||
std::vector<json> output;
|
||||
@@ -741,7 +741,7 @@ json server_task_result_cmpl_final::to_json_anthropic() {
|
||||
msg = oaicompat_msg;
|
||||
} else {
|
||||
msg.role = "assistant";
|
||||
msg.content = content;
|
||||
msg.content = content.text;
|
||||
}
|
||||
|
||||
// thinking block comes first (Anthropic extended thinking format)
|
||||
@@ -1050,7 +1050,7 @@ json server_task_result_cmpl_partial::to_json_non_oaicompat() {
|
||||
// non-OAI-compat JSON
|
||||
json res = json {
|
||||
{"index", index},
|
||||
{"content", content},
|
||||
{"content", content.text},
|
||||
{"tokens", tokens},
|
||||
{"stop", false},
|
||||
{"id_slot", id_slot},
|
||||
@@ -1081,7 +1081,7 @@ json server_task_result_cmpl_partial::to_json_oaicompat() {
|
||||
json res = json {
|
||||
{"choices", json::array({
|
||||
json{
|
||||
{"text", content},
|
||||
{"text", content.text},
|
||||
{"index", index},
|
||||
{"logprobs", logprobs},
|
||||
{"finish_reason", nullptr},
|
||||
@@ -1316,7 +1316,7 @@ json server_task_result_cmpl_partial::to_json_oaicompat_resp() {
|
||||
json server_task_result_cmpl_partial::to_json_oaicompat_asr() {
|
||||
json event = json {
|
||||
{"type", "transcript.text.delta"},
|
||||
{"delta", content},
|
||||
{"delta", content.text},
|
||||
};
|
||||
return event;
|
||||
}
|
||||
|
||||
@@ -108,7 +108,7 @@ struct task_result_state {
|
||||
std::vector<common_chat_msg_diff> diffs;
|
||||
common_chat_parser_params chat_parser_params;
|
||||
common_chat_msg chat_msg;
|
||||
std::string generated_text; // append new chunks of generated text here
|
||||
common_chat_input generated_input; // append new chunks of generated text here
|
||||
std::vector<std::string> generated_tool_call_ids;
|
||||
std::unordered_set<size_t> sent_tool_call_names;
|
||||
|
||||
@@ -128,7 +128,7 @@ struct task_result_state {
|
||||
|
||||
// parse partial tool calls and update the internal state
|
||||
common_chat_msg update_chat_msg(
|
||||
const std::string & text_added,
|
||||
const common_chat_input & added,
|
||||
bool is_partial,
|
||||
std::vector<common_chat_msg_diff> & diffs,
|
||||
bool filter_tool_calls = false);
|
||||
@@ -350,7 +350,7 @@ struct completion_token_output {
|
||||
};
|
||||
|
||||
struct server_task_result_cmpl_final : server_task_result {
|
||||
std::string content;
|
||||
common_chat_input content;
|
||||
llama_tokens tokens;
|
||||
|
||||
bool stream;
|
||||
@@ -425,8 +425,8 @@ struct server_task_result_cmpl_final : server_task_result {
|
||||
};
|
||||
|
||||
struct server_task_result_cmpl_partial : server_task_result {
|
||||
std::string content;
|
||||
llama_tokens tokens;
|
||||
common_chat_input content;
|
||||
llama_tokens tokens;
|
||||
|
||||
int32_t n_decoded;
|
||||
int32_t n_prompt_tokens;
|
||||
|
||||
Reference in new issue
Block a user