diff --git a/common/chat.cpp b/common/chat.cpp index 1c1533325..2c795c281 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1452,14 +1452,70 @@ common_chat_params common_chat_templates_apply(const struct common_chat_template common_chat_templates_apply_legacy(tmpls, inputs); } -common_chat_msg common_chat_parse(const std::string & input, +void common_chat_input::append(const std::string & piece, llama_token token) { + if (piece.empty()) { + return; + } + tokens.push_back(token); + tokens.resize(tokens.size() + piece.size() - 1, LLAMA_TOKEN_NULL); + text += piece; +} + +void common_chat_input::append(const common_chat_input & chunk) { + tokens.insert(tokens.end(), chunk.tokens.begin(), chunk.tokens.end()); + text += chunk.text; +} + +void common_chat_input::truncate(size_t pos) { + if (pos < text.size()) { + text.erase(pos); + tokens.resize(pos); + } +} + +common_chat_input common_chat_input::substr(size_t pos, size_t n) const { + common_chat_input out; + out.text = text.substr(pos, n); + out.tokens.assign(tokens.begin() + pos, tokens.begin() + pos + out.size()); + return out; +} + +void common_chat_input::prepend(const std::string & prefix) { + tokens.insert(tokens.begin(), prefix.size(), LLAMA_TOKEN_NULL); + text = prefix + text; +} + +void common_chat_input::prepend(const common_chat_input & prefix) { + tokens.insert(tokens.begin(), prefix.tokens.begin(), prefix.tokens.end()); + text = prefix.text + text; +} + +common_chat_input common_chat_input_tokenize(const llama_vocab * vocab, const std::string & text) { + common_chat_input input; + auto tokens = common_tokenize(vocab, text, false, true); + for (size_t i = 0; i < tokens.size(); i++) { + std::string piece = common_token_to_piece(vocab, tokens[i], true); + if (i == 0 && std::isspace(piece[0]) && !std::isspace(text[0])) { + // Some tokenizers will add a space before the first special token, need to exclude + continue; + } + input.append(piece, tokens[i]); + } + if (input.text != text) { + // the pieces do not give back the same text, keep the text without tokens + return common_chat_input(text); + } + return input; +} + +common_chat_msg common_chat_parse(const common_chat_input & input, bool is_partial, const common_chat_parser_params & params) { return common_chat_peg_parse(params.parser, input, is_partial, params); } common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, - const std::string & input, + const common_chat_input & input, bool is_partial, const common_chat_parser_params & params) { const common_peg_arena & parser = src_parser.empty() ? @@ -1470,18 +1526,17 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars LOG_DBG("No parser definition detected, assuming pure content parser."); } - const std::string effective_input = params.generation_prompt.empty() - ? input - : params.generation_prompt + input; + common_chat_input effective_input = input; + effective_input.prepend(params.generation_prompt); - //LOG_DBG("Parsing PEG input with format %s: %s\n", common_chat_format_name(params.format), effective_input.c_str()); + //LOG_DBG("Parsing PEG input with format %s: %s\n", common_chat_format_name(params.format), effective_input.text.c_str()); common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_LENIENT; if (params.debug) { flags |= COMMON_PEG_PARSE_FLAG_DEBUG; } - common_peg_parse_context ctx(effective_input, flags); + common_peg_parse_context ctx(std::move(effective_input.text), std::move(effective_input.tokens), flags); auto result = parser.parse(ctx); if (result.fail()) { @@ -1507,8 +1562,8 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars } return msg; } - LOG_WRN("%s: unparsed %s output: %s\n", __func__, common_chat_format_name(params.format), effective_input.substr(result.end).c_str()); - LOG_DBG("%s: full %s output triggering error:\n=== BEGIN ===\n%s\n=== END ===\n", __func__, common_chat_format_name(params.format), effective_input.c_str()); + LOG_WRN("%s: unparsed %s output: %s\n", __func__, common_chat_format_name(params.format), ctx.input.substr(result.end).c_str()); + LOG_DBG("%s: full %s output triggering error:\n=== BEGIN ===\n%s\n=== END ===\n", __func__, common_chat_format_name(params.format), ctx.input.c_str()); throw std::runtime_error(std::string("The model produced output that does not match the expected ") + common_chat_format_name(params.format) + " format"); } diff --git a/common/chat.h b/common/chat.h index 0e1423a5a..4509a04cc 100644 --- a/common/chat.h +++ b/common/chat.h @@ -282,6 +282,31 @@ struct common_chat_params { common_chat_msg_delimiters message_delimiters; }; +struct common_chat_input { + std::string text; + std::vector tokens; + + common_chat_input() = default; + + // plain text, with no tokens + explicit common_chat_input(std::string text) : text(std::move(text)), tokens(this->text.size(), LLAMA_TOKEN_NULL) {} + + size_t size() const { return text.size(); } + bool empty() const { return text.empty(); } + + void append(const std::string & piece, llama_token token); + void append(const common_chat_input & chunk); + + void prepend(const std::string & prefix); + void prepend(const common_chat_input & prefix); + + void truncate(size_t pos); + + common_chat_input substr(size_t pos, size_t n = std::string::npos) const; +}; + +common_chat_input common_chat_input_tokenize(const llama_vocab * vocab, const std::string & text); + // per-message parsing syntax // should be derived from common_chat_params struct common_chat_parser_params { @@ -289,7 +314,7 @@ struct common_chat_parser_params { common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_NONE; // TODO: refactor this to "bool parse_reasoning" // Whether reasoning_content should be inlined in the content (e.g. for reasoning_format=deepseek in stream mode) bool reasoning_in_content = false; - std::string generation_prompt; + common_chat_input generation_prompt; bool parse_tool_calls = true; bool is_continuation = false; bool echo = false; // Include assistant prefilled msg in output @@ -298,7 +323,7 @@ struct common_chat_parser_params { common_chat_parser_params() = default; common_chat_parser_params(const common_chat_params & chat_params) { format = chat_params.format; - generation_prompt = chat_params.generation_prompt; + generation_prompt = common_chat_input(chat_params.generation_prompt); } }; @@ -337,8 +362,8 @@ std::string common_chat_format_example(const struct common_chat_templates * const std::map & chat_template_kwargs); const char * common_chat_format_name(common_chat_format format); -common_chat_msg common_chat_parse(const std::string & input, bool is_partial, const common_chat_parser_params & params); -common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, const std::string & input, bool is_partial, const common_chat_parser_params & params); +common_chat_msg common_chat_parse(const common_chat_input & input, bool is_partial, const common_chat_parser_params & params); +common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, const common_chat_input & input, bool is_partial, const common_chat_parser_params & params); // used by arg and server const char * common_reasoning_format_name(common_reasoning_format format); diff --git a/common/peg-parser.h b/common/peg-parser.h index 888325945..816d007b0 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -2,6 +2,7 @@ #include "json-schema.h" #include "json.h" +#include "llama.h" #include #include @@ -182,7 +183,8 @@ inline common_peg_parse_flags operator~(common_peg_parse_flags a) { } struct common_peg_parse_context { - std::string input; + std::string input; // [h, e, l, l, o, _, w, o, r, l, d] + std::vector tokens; // [id, -1, -1, -1, -1, id, -1, -1, -1, -1, -1] common_peg_parse_flags flags; common_peg_ast_arena ast; @@ -194,6 +196,11 @@ struct common_peg_parse_context { common_peg_parse_context(const std::string & input, common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_NONE) : input(input), flags(flags), parse_depth(0) {} + common_peg_parse_context(std::string input, std::vector tokens, common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_NONE) + : input(std::move(input)), tokens(std::move(tokens)), flags(flags), parse_depth(0) { + GGML_ASSERT(this->tokens.empty() || this->tokens.size() == this->input.size()); + } + bool is_lenient() const { return flags & COMMON_PEG_PARSE_FLAG_LENIENT; } bool is_debug() const { return flags & COMMON_PEG_PARSE_FLAG_DEBUG; } }; diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 9f96e1084..e31c093b0 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -1105,7 +1105,7 @@ struct make_peg_parser { common_chat_msg parse(const std::string & msg, bool is_partial) const { common_chat_parser_params parser_params(params_); parser_params.debug = detailed_debug_; - return common_chat_peg_parse(arena_, msg, is_partial, parser_params); + return common_chat_peg_parse(arena_, common_chat_input(msg), is_partial, parser_params); } }; @@ -4643,7 +4643,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) { bool got_out_of_range = false; std::string error_msg; try { - common_chat_peg_parse(arena, bad_input, /*is_partial=*/false, pp); + common_chat_peg_parse(arena, common_chat_input(bad_input), /*is_partial=*/false, pp); } catch (const std::out_of_range & e) { got_out_of_range = true; error_msg = e.what(); diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index d1af984e6..23835e185 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -267,7 +267,7 @@ struct server_slot { bool spec_is_replay = false; std::mt19937 spec_synth_rng; - // TODO: move members that belong to the task (such as `generated_text`, `has_new_line`) to task_results_state + // TODO: move members that belong to the task (such as `generated`, `has_new_line`) to task_results_state // see https://github.com/ggml-org/llama.cpp/pull/18283#issuecomment-3710175837 std::unique_ptr task; std::unique_ptr task_prev; // used for debugging @@ -285,7 +285,7 @@ struct server_slot { size_t last_nl_pos = 0; - std::string generated_text; + common_chat_input generated; std::string debug_generated_text; llama_tokens generated_tokens; size_t n_sent_text = 0; // number of sent text character (i.e. handle partial UTF-8 on streaming) @@ -381,7 +381,7 @@ struct server_slot { spec_is_replay = false; last_nl_pos = 0; - generated_text = ""; + generated = {}; has_new_line = false; truncated = false; stop = STOP_TYPE_NONE; @@ -750,7 +750,7 @@ struct server_slot { if (!only_metrics) { res["prompt"] = ptask->tokens.detokenize(ctx_tgt, true); - res["generated"] = generated_text.empty() ? debug_generated_text : generated_text; + res["generated"] = generated.empty() ? debug_generated_text : generated.text; } } @@ -1968,28 +1968,26 @@ private: const std::string token_str = result.text_to_send; slot.sampled = result.tok; - slot.generated_text += token_str; + slot.generated.append(token_str, result.tok); if (slot.task->params.return_tokens) { slot.generated_tokens.push_back(result.tok); } slot.has_next_token = true; // check if there is incomplete UTF-8 character at the end - bool incomplete = validate_utf8(slot.generated_text) < slot.generated_text.size(); + bool incomplete = validate_utf8(slot.generated.text) < slot.generated.text.size(); // search stop word and delete it if (!incomplete) { - size_t pos = std::min(slot.n_sent_text, slot.generated_text.size()); + size_t pos = std::min(slot.n_sent_text, slot.generated.text.size()); - const std::string str_test = slot.generated_text.substr(pos); + const std::string str_test = slot.generated.text.substr(pos); bool send_text = true; size_t stop_pos = slot.find_stopping_strings(str_test, token_str.size(), true); if (stop_pos != std::string::npos) { - slot.generated_text.erase( - slot.generated_text.begin() + pos + stop_pos, - slot.generated_text.end()); - pos = std::min(slot.n_sent_text, slot.generated_text.size()); + slot.generated.truncate(pos + stop_pos); + pos = std::min(slot.n_sent_text, slot.generated.text.size()); } else if (slot.has_next_token && !llama_vocab_is_eog(vocab, result.tok) ) { stop_pos = slot.find_stopping_strings(str_test, token_str.size(), false); send_text = stop_pos == std::string::npos; @@ -1998,7 +1996,7 @@ private: // check if there is any token to predict if (send_text) { // no send the stop word in the response - result.text_to_send = slot.generated_text.substr(pos, std::string::npos); + result.text_to_send = slot.generated.text.substr(pos, std::string::npos); slot.n_sent_text += result.text_to_send.size(); // add the token to slot queue and cache } else { @@ -2042,17 +2040,17 @@ private: size_t pos = slot.last_nl_pos; int n_indent = 0; - while (pos < slot.generated_text.size() && (slot.generated_text[pos] == ' ' || slot.generated_text[pos] == '\t')) { + while (pos < slot.generated.text.size() && (slot.generated.text[pos] == ' ' || slot.generated.text[pos] == '\t')) { n_indent++; pos++; } - if (pos < slot.generated_text.size() && n_indent < slot.task->params.n_indent) { + if (pos < slot.generated.text.size() && n_indent < slot.task->params.n_indent) { slot.stop = STOP_TYPE_LIMIT; slot.has_next_token = false; // cut the last line - slot.generated_text.erase(pos, std::string::npos); + slot.generated.truncate(pos); SLT_DBG(slot, "stopped by indentation limit, n_gen = %d, n_indent = %d\n", (int) slot.stats.n_gen, n_indent); } @@ -2060,7 +2058,7 @@ private: // find the next new line { - const size_t pos = slot.generated_text.find('\n', slot.last_nl_pos); + const size_t pos = slot.generated.text.find('\n', slot.last_nl_pos); if (pos != std::string::npos) { slot.last_nl_pos = pos + 1; @@ -2192,7 +2190,7 @@ private: if (is_begin) { res->is_begin = true; } else { - res->content = tkn.text_to_send; + res->content = slot.generated.substr(slot.n_sent_text - tkn.text_to_send.size(), tkn.text_to_send.size()); res->tokens.assign(1, tkn.tok); } @@ -2229,15 +2227,15 @@ private: // keep copy of last generated text for debugging purposes if (slots_debug) { - slot.debug_generated_text = slot.generated_text; + slot.debug_generated_text = slot.generated.text; } // in stream mode, content and tokens are already in last partial chunk if (slot.task->params.stream) { - res->content = ""; + res->content = {}; res->tokens = llama_tokens{}; } else { - res->content = std::move(slot.generated_text); + res->content = std::move(slot.generated); res->tokens = std::move(slot.generated_tokens); } res->stats = slot.stats; diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cpp index 27ecafb7a..c7d0bc4de 100644 --- a/tools/server/server-schema.cpp +++ b/tools/server/server-schema.cpp @@ -311,8 +311,14 @@ std::vector> make_llama_cmpl_schema(const common_params & ->set_desc("Generation prompt appended to the chat template output") ->set_handler([&](field_eval_context & ctx, const json & data) { std::string s = data.at("generation_prompt").get(); - ctx.params.chat_parser_params.generation_prompt = s; ctx.params.sampling.generation_prompt = s; + + if (ctx.vocab == nullptr) { + ctx.params.chat_parser_params.generation_prompt = common_chat_input(s); + return; + } + + ctx.params.chat_parser_params.generation_prompt = common_chat_input_tokenize(ctx.vocab, s); })); add((new field_bool("parse_tool_calls", params.chat_parser_params.parse_tool_calls)) diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index a5c33c056..21c1fe4e1 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -76,7 +76,7 @@ json task_params::to_json(bool only_metrics) const { {"chat_format", common_chat_format_name(chat_parser_params.format)}, {"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)}, {"reasoning_in_content", chat_parser_params.reasoning_in_content}, - {"generation_prompt", chat_parser_params.generation_prompt}, + {"generation_prompt", chat_parser_params.generation_prompt.text}, {"samplers", samplers}, {"speculative.types", common_speculative_type_name_str(speculative.types)}, {"timings_per_token", timings_per_token}, @@ -135,7 +135,7 @@ json task_params::to_json(bool only_metrics) const { {"chat_format", common_chat_format_name(chat_parser_params.format)}, {"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)}, {"reasoning_in_content", chat_parser_params.reasoning_in_content}, - {"generation_prompt", chat_parser_params.generation_prompt}, + {"generation_prompt", chat_parser_params.generation_prompt.text}, {"samplers", samplers}, {"speculative.types", common_speculative_type_name_str(speculative.types)}, {"timings_per_token", timings_per_token}, @@ -155,20 +155,20 @@ task_result_state::task_result_state(const common_chat_parser_params & chat_pars , oai_resp_message_id("msg_" + random_string()) { if (chat_parser_params.is_continuation && !chat_parser_params.echo) { // initialize chat_msg to avoid emitting a delta containing the assistant prefill - chat_msg = common_chat_parse("", true, chat_parser_params); + chat_msg = common_chat_parse(generated_input, true, chat_parser_params); } } common_chat_msg task_result_state::update_chat_msg( - const std::string & text_added, + const common_chat_input & added, bool is_partial, std::vector & diffs, bool filter_tool_calls) { - generated_text += text_added; + generated_input.append(added); auto msg_prv_copy = chat_msg; - //SRV_DBG("Parsing chat message: %s\n", generated_text.c_str()); + //SRV_DBG("Parsing chat message: %s\n", generated_input.text.c_str()); auto new_msg = common_chat_parse( - generated_text, + generated_input, is_partial, chat_parser_params); if (!new_msg.empty()) { @@ -340,7 +340,7 @@ json server_task_result_cmpl_final::to_json() { json server_task_result_cmpl_final::to_json_non_oaicompat() { json res = json { {"index", index}, - {"content", content}, + {"content", content.text}, {"tokens", tokens}, {"id_slot", id_slot}, {"stop", true}, @@ -386,7 +386,7 @@ json server_task_result_cmpl_final::to_json_oaicompat() { json res = json { {"choices", json::array({ json{ - {"text", content}, + {"text", content.text}, {"index", index}, {"logprobs", logprobs}, {"finish_reason", finish_reason}, @@ -418,7 +418,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat() { msg = oaicompat_msg; } else { msg.role = "assistant"; - msg.content = content; + msg.content = content.text; } if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) { finish_reason = msg.tool_calls.empty() ? "stop" : "tool_calls"; @@ -531,7 +531,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp() { msg = oaicompat_msg; } else { msg.role = "assistant"; - msg.content = content; + msg.content = content.text; } std::vector output; @@ -741,7 +741,7 @@ json server_task_result_cmpl_final::to_json_anthropic() { msg = oaicompat_msg; } else { msg.role = "assistant"; - msg.content = content; + msg.content = content.text; } // thinking block comes first (Anthropic extended thinking format) @@ -1050,7 +1050,7 @@ json server_task_result_cmpl_partial::to_json_non_oaicompat() { // non-OAI-compat JSON json res = json { {"index", index}, - {"content", content}, + {"content", content.text}, {"tokens", tokens}, {"stop", false}, {"id_slot", id_slot}, @@ -1081,7 +1081,7 @@ json server_task_result_cmpl_partial::to_json_oaicompat() { json res = json { {"choices", json::array({ json{ - {"text", content}, + {"text", content.text}, {"index", index}, {"logprobs", logprobs}, {"finish_reason", nullptr}, @@ -1316,7 +1316,7 @@ json server_task_result_cmpl_partial::to_json_oaicompat_resp() { json server_task_result_cmpl_partial::to_json_oaicompat_asr() { json event = json { {"type", "transcript.text.delta"}, - {"delta", content}, + {"delta", content.text}, }; return event; } diff --git a/tools/server/server-task.h b/tools/server/server-task.h index 752595b0f..552c03b07 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -108,7 +108,7 @@ struct task_result_state { std::vector diffs; common_chat_parser_params chat_parser_params; common_chat_msg chat_msg; - std::string generated_text; // append new chunks of generated text here + common_chat_input generated_input; // append new chunks of generated text here std::vector generated_tool_call_ids; std::unordered_set sent_tool_call_names; @@ -128,7 +128,7 @@ struct task_result_state { // parse partial tool calls and update the internal state common_chat_msg update_chat_msg( - const std::string & text_added, + const common_chat_input & added, bool is_partial, std::vector & diffs, bool filter_tool_calls = false); @@ -350,7 +350,7 @@ struct completion_token_output { }; struct server_task_result_cmpl_final : server_task_result { - std::string content; + common_chat_input content; llama_tokens tokens; bool stream; @@ -425,8 +425,8 @@ struct server_task_result_cmpl_final : server_task_result { }; struct server_task_result_cmpl_partial : server_task_result { - std::string content; - llama_tokens tokens; + common_chat_input content; + llama_tokens tokens; int32_t n_decoded; int32_t n_prompt_tokens;