diff --git a/common/chat-auto-parser-generator.cpp b/common/chat-auto-parser-generator.cpp index e59a556ad..220889367 100644 --- a/common/chat-auto-parser-generator.cpp +++ b/common/chat-auto-parser-generator.cpp @@ -373,7 +373,9 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte auto & p = ctx.p; const auto & inputs = ctx.inputs; - auto until_suffix = p.rule("until-suffix", p.until(arguments.value_suffix)); + // A raw string value needs a suffix to end on, without one it is parsed as JSON + bool raw_strings = !arguments.value_suffix.empty(); + auto until_suffix = raw_strings ? p.rule("until-suffix", p.until(arguments.value_suffix)) : p.eps(); common_peg_parser tool_choice = p.choice(); @@ -389,9 +391,9 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte p.tool_arg(p.tool_arg_open(arguments.name_prefix + p.tool_arg_name(p.literal(param.name)) + arguments.name_suffix) + arguments.value_prefix + - (param.schema->may_be_string() ? - p.ac(p.tool_arg_string_value(until_suffix) + - p.tool_arg_close(p.literal(arguments.value_suffix)), arguments.value_suffix) : + (raw_strings && param.schema->may_be_string() ? + p.through(p.tool_arg_string_value(until_suffix), + p.tool_arg_close(p.literal(arguments.value_suffix))) : (p.tool_arg_json_value(p.schema( p.json(), "tool-" + std::to_string(tool_index) + "-arg-" + std::to_string(param_index) + "-schema", doc, *param.schema)) + p.tool_arg_close(p.literal(arguments.value_suffix))))); diff --git a/common/chat-auto-parser.h b/common/chat-auto-parser.h index 8ae15c91e..696b9e5e6 100644 --- a/common/chat-auto-parser.h +++ b/common/chat-auto-parser.h @@ -71,6 +71,7 @@ struct generation_params { bool is_inference = true; bool add_inference = false; bool mark_input = true; // whether to mark input strings in the jinja context + common_peg_tokens specials; bool has_continuation() const { return continue_final_message != COMMON_CHAT_CONTINUATION_NONE && !continue_msg.empty(); diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index c77ec117b..b39607e5c 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -71,6 +71,9 @@ constexpr size_t COMMON_CHAT_MAX_PERMUTE = 6; class common_chat_peg_builder : public common_peg_parser_builder { public: + common_chat_peg_builder() = default; + explicit common_chat_peg_builder(common_peg_tokens tokens) : common_peg_parser_builder(std::move(tokens)) {} + // Tag constants (from former common_chat_peg_base_builder) static constexpr const char * REASONING_BLOCK = "reasoning-block"; static constexpr const char * REASONING = "reasoning"; @@ -200,6 +203,14 @@ inline common_peg_arena build_chat_peg_parser( return builder.build(); } +inline common_peg_arena build_chat_peg_parser( + const common_peg_tokens & specials, + const std::function & fn) { + common_chat_peg_builder builder(specials); + builder.set_root(fn(builder)); + return builder.build(); +} + class tag_based_peg_mapper { public: std::map tags; diff --git a/common/chat.cpp b/common/chat.cpp index de5f1c490..3cff920cf 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -315,6 +315,7 @@ struct common_chat_templates { bool add_bos; bool add_eos; bool has_explicit_template; // Model had builtin template or template overridden was specified. + common_peg_tokens specials; std::unique_ptr template_default; // always set (defaults to chatml) std::unique_ptr template_tool_use; std::unique_ptr template_tts; // the prompt of a text to speak @@ -802,6 +803,10 @@ void common_chat_templates_free(struct common_chat_templates * tmpls) { delete tmpls; } +void common_chat_templates_set_specials(struct common_chat_templates * tmpls, common_peg_tokens specials) { + tmpls->specials = std::move(specials); +} + bool common_chat_templates_was_explicit(const struct common_chat_templates * tmpls) { return tmpls->has_explicit_template; } @@ -978,6 +983,7 @@ common_chat_templates_ptr common_chat_templates_init(const struct llama_model * std::string token_eos = eos_token_override; bool add_bos = false; bool add_eos = false; + common_peg_tokens specials; if (model) { const auto * vocab = llama_model_get_vocab(model); const auto get_token = [&](llama_token token, const char * name, const char * jinja_variable_name) { @@ -997,11 +1003,13 @@ common_chat_templates_ptr common_chat_templates_init(const struct llama_model * token_eos = get_token(llama_vocab_eos(vocab), "EOS", "eos_token"); add_bos = llama_vocab_get_add_bos(vocab); add_eos = llama_vocab_get_add_eos(vocab); + specials = common_peg_tokens(vocab); } common_chat_templates_ptr tmpls(new common_chat_templates()); tmpls->has_explicit_template = has_explicit_template; tmpls->add_bos = add_bos; tmpls->add_eos = add_eos; + tmpls->specials = std::move(specials); try { tmpls->template_default = std::make_unique(default_template_src, token_bos, token_eos); } catch (const std::exception & e) { @@ -1543,6 +1551,7 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_ params.add_generation_prompt = inputs.add_generation_prompt; params.add_bos = tmpls->add_bos; params.add_eos = tmpls->add_eos; + params.specials = tmpls->specials; params.continue_final_message = inputs.continue_final_message; if (params.continue_final_message != COMMON_CHAT_CONTINUATION_NONE) { @@ -1825,6 +1834,7 @@ common_chat_input common_chat_input_tokenize(const llama_vocab * vocab, const st } if (input.text != text) { // the pieces do not give back the same text, keep the text without tokens + LOG_WRN("%s: token pieces do not match the text, parsing without token ids\n", __func__); return common_chat_input(text); } return input; @@ -1935,11 +1945,17 @@ common_chat_session::common_chat_session(const common_chat_templates * tm thinking_ends = std::move(applied.thinking_end_tags); parser_params.format = applied.format; - parser_params.generation_prompt = vocab ? common_chat_input_tokenize(vocab, applied.generation_prompt) - : common_chat_input(applied.generation_prompt); parser_params.debug = params.debug; parser_params.parser = std::move(applied.parser); + if (vocab) { + parser_params.generation_prompt = common_chat_input_tokenize(vocab, applied.generation_prompt); + } else if (params.tokenize) { + parser_params.generation_prompt = params.tokenize(applied.generation_prompt); + } else { + parser_params.generation_prompt = common_chat_input(applied.generation_prompt); + } + delimiters = std::move(applied.message_delimiters); if (vocab) { diff --git a/common/chat.h b/common/chat.h index 43331b16d..ca324d812 100644 --- a/common/chat.h +++ b/common/chat.h @@ -372,6 +372,9 @@ common_chat_templates_ptr common_chat_templates_init(const struct llama_model * const std::string & bos_token_override = "", const std::string & eos_token_override = ""); +// override special tokens for testing +void common_chat_templates_set_specials(struct common_chat_templates * tmpls, common_peg_tokens specials); + bool common_chat_templates_was_explicit(const struct common_chat_templates * tmpls); std::string common_chat_templates_source(const struct common_chat_templates * tmpls, const std::string & variant = ""); @@ -397,6 +400,9 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, const struct common_chat_session_params { bool echo = false; // include the assistant prefill in the output when continuing a message bool debug = false; // enable debug output for the PEG parser + + // tokenize the generation prompt when there is no vocab for testing + std::function tokenize; }; class common_chat_session { diff --git a/common/json-schema-to-grammar.cpp b/common/json-schema-to-grammar.cpp index b6ea61fc3..4690a57fa 100644 --- a/common/json-schema-to-grammar.cpp +++ b/common/json-schema-to-grammar.cpp @@ -676,8 +676,8 @@ private: const auto & node = trie.nodes[idx]; std::string rejects; auto first = true; - for (const auto & [cpt, child] : node.children) { - std::string c = common_unicode_cpt_to_utf8(cpt); + for (const auto & [sym, child] : node.children) { + std::string c = common_unicode_cpt_to_utf8(sym.value); rejects += c; if (first) { first = false; diff --git a/common/parsers/gemma4.cpp b/common/parsers/gemma4.cpp index 2a2e929e1..ff4dfad67 100644 --- a/common/parsers/gemma4.cpp +++ b/common/parsers/gemma4.cpp @@ -24,7 +24,9 @@ struct depth_guard { // Gemma 4 value parsers void gemma4_value_rules(common_chat_peg_builder & p) { // The tags are what gemma4_to_json() in the mapper reads, the rules may wrap them freely. - p.rule("gemma4-string", p.tag(gemma4::STRING, p.literal("<|\"|>") + p.tag(gemma4::STRING_CONTENT, p.until("<|\"|>")) + p.literal("<|\"|>"))); + auto string_delim = p.token("<|\"|>"); + + p.rule("gemma4-string", p.tag(gemma4::STRING, string_delim + p.tag(gemma4::STRING_CONTENT, p.until(string_delim)) + string_delim)); p.rule("gemma4-bool", p.tag(gemma4::BOOL, p.json_bool())); @@ -34,7 +36,7 @@ void gemma4_value_rules(common_chat_peg_builder & p) { // Keys are bare in the template, but Gemma 4 occasionally wraps them in quotes p.rule("gemma4-dict-key", p.tag(gemma4::DICT_KEY, p.choice({ - p.literal("<|\"|>") + p.tag(gemma4::DICT_KEY_NAME, p.until("<|\"|>")) + p.literal("<|\"|>"), + string_delim + p.tag(gemma4::DICT_KEY_NAME, p.until(string_delim)) + string_delim, p.literal("\"") + p.tag(gemma4::DICT_KEY_NAME, p.chars("[^\"]", 1, -1)) + p.literal("\""), p.literal("'") + p.tag(gemma4::DICT_KEY_NAME, p.chars("[^']", 1, -1)) + p.literal("'"), p.tag(gemma4::DICT_KEY_NAME, p.chars("[^:}]", 1, -1)), @@ -144,7 +146,7 @@ class gemma4_schema_parser { alts |= name(); } // Gemma 4 may emit keys in its string delimiters or quotes - alts |= p_.literal("<|\"|>") + name() + p_.literal("<|\"|>"); + alts |= p_.token("<|\"|>") + name() + p_.token("<|\"|>"); if (key.find('\'') == std::string::npos) { alts |= p_.literal("'") + name() + p_.literal("'"); } @@ -331,18 +333,23 @@ common_chat_params common_chat_params_init_gemma4(const common_chat_template & auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto start = p.rule("start", p.optional(p.literal("<|turn>model\n"))); + data.parser = build_chat_peg_parser(inputs.specials, [&](common_chat_peg_builder & p) { + auto start = p.rule("start", p.optional(p.token("<|turn>") + p.literal("model\n"))); + auto channel_start = p.token("<|channel>"); + auto channel_end = p.token(""); + auto think_start = channel_start + p.literal("thought"); + auto tool_call_start = p.token("<|tool_call>"); + // the model may start a tool call before closing the thought channel + auto think_end = channel_end | p.peek(tool_call_start); if (extract_reasoning) { - // the model may start a tool call before closing the thought channel - p.rule("thought", p.literal("<|channel>thought") + p.space() + p.reasoning(p.until_one_of({"", "<|tool_call>"})) + (p.literal("") | p.peek(p.literal("<|tool_call>")))); + p.rule("thought", think_start + p.space() + p.reasoning(p.until(channel_end | tool_call_start)) + think_end); } else { - p.rule("thought", p.content(p.literal("<|channel>thought") + p.space() + p.until_one_of({"", "<|tool_call>"}) + (p.literal("") | p.peek(p.literal("<|tool_call>"))))); + p.rule("thought", p.content(think_start + p.space() + p.until(channel_end | tool_call_start) + think_end)); } - auto consume_empty_channels = p.gbnf(p.zero_or_more(p.literal("<|channel>") + p.negate(p.literal("thought"))), ""); - auto thought = (p.peek(p.literal("<|channel>")) + consume_empty_channels + p.ref("thought")) | p.negate(p.literal("<|channel>")); + auto consume_empty_channels = p.gbnf(p.zero_or_more(channel_start + p.negate(p.literal("thought"))), ""); + auto thought = (p.peek(channel_start) + consume_empty_channels + p.ref("thought")) | p.negate(channel_start); if (has_response_format) { auto response_format = p.literal("```json") << @@ -378,39 +385,35 @@ common_chat_params common_chat_params_init_gemma4(const common_chat_template & }))); }); - auto tool_call = p.trigger_rule("tool-call", p.repeat( - "<|tool_call>call:" + tool_choice + "", - /* min = */ inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0, - /* max = */ inputs.parallel_tool_calls ? -1 : 1 - )); + auto tool_call_rest = p.literal("call:") + tool_choice + p.token(""); + auto tool_call = p.rule("tool-call", tool_call_start + tool_call_rest); + auto more = inputs.parallel_tool_calls ? p.zero_or_more(tool_call) : p.eps(); + + auto tool_calls = p.trigger_rule("tool-calls", tool_call_start, tool_call_rest + more); if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) { - return start + thought + tool_call; + return start + thought + tool_calls; } - auto scan_to_toolcall = p.rule("scan-to-toolcall", p.until("<|tool_call>")); - auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", "", "<|tool_call>"}))); + auto scan_to_toolcall = p.rule("scan-to-toolcall", p.until(tool_call_start)); + auto content = p.rule("content", p.content(p.until(channel_start | channel_end | tool_call_start))); auto message = p.rule("message", thought + content); - return start + p.zero_or_more(message) + scan_to_toolcall + tool_call; + return start + p.zero_or_more(message) + scan_to_toolcall + p.optional(tool_calls); } // Gemma 4 may emit an extra <|channel>thought\n at the end of the content. It may // also emit a single trailing token. Consume all complete reasoning blocks and // then stop at the first unmatched token. - auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", ""}))); + auto content = p.rule("content", p.content(p.until(channel_start | channel_end))); auto message = p.rule("message", thought + content); return start + p.one_or_more(message); }); if (include_grammar) { - data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); + data.grammar_lazy = false; data.grammar = build_grammar([&](const common_grammar_builder & builder) { - data.parser.build_grammar(builder, data.grammar_lazy); + data.parser.build_grammar(builder, !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED))); }); - - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<|tool_call>" }, - }; } return data; diff --git a/common/parsers/gpt-oss.cpp b/common/parsers/gpt-oss.cpp index 4cbf929f9..fbc388c97 100644 --- a/common/parsers/gpt-oss.cpp +++ b/common/parsers/gpt-oss.cpp @@ -65,17 +65,18 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto start = p.rule("start", p.literal("<|start|>assistant")); - auto end = p.rule("end", p.literal("<|end|>")); - auto content = p.rule("message-content", p.until("<|end|>")); - auto channel = p.literal("<|channel|>") + (p.literal("commentary") | p.literal("analysis")); + data.parser = build_chat_peg_parser(inputs.specials, [&](common_chat_peg_builder & p) { + auto start_assistant = p.token("<|start|>") + p.literal("assistant"); + auto start = p.rule("start", start_assistant); + auto end = p.rule("end", p.token("<|end|>")); + auto content = p.rule("message-content", p.until(p.token("<|end|>"))); + auto channel = p.token("<|channel|>") + (p.literal("commentary") | p.literal("analysis")); auto constrain_type = p.chars("[A-Za-z0-9_-]", 1, -1); - auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type); + auto constraint = p.optional(p.chars("[ ]", 1, -1) + p.optional(p.token("<|constrain|>")) + constrain_type); // Occasionally, gpt-oss-20b will prefix channels with this commentary - auto stray_commentary = p.optional(p.literal("<|channel|>commentary") + p.optional(p.literal(" to=assistant"))); - auto start_analysis = stray_commentary + p.literal("<|channel|>analysis<|message|>"); + auto stray_commentary = p.optional(p.token("<|channel|>") + p.literal("commentary") + p.optional(p.literal(" to=assistant"))); + auto start_analysis = stray_commentary + p.token("<|channel|>") + p.literal("analysis") + p.token("<|message|>"); if (extract_reasoning) { p.rule("analysis", start_analysis + p.reasoning(content) + end); @@ -84,8 +85,8 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & } auto analysis = p.ref("analysis"); - auto preamble = p.rule("preamble", p.literal("<|channel|>commentary<|message|>") + p.content(content) + end); - auto final_msg = p.rule("final", stray_commentary + p.literal("<|channel|>final") + constraint + p.literal("<|message|>") + p.content(content)); + auto preamble = p.rule("preamble", p.token("<|channel|>") + p.literal("commentary") + p.token("<|message|>") + p.content(content) + end); + auto final_msg = p.rule("final", stray_commentary + p.token("<|channel|>") + p.literal("final") + constraint + p.token("<|message|>") + p.content(content)); // Consume any unsolicited tool calls, e.g. builtin functions auto unsolicited = p.rule("unsolicited", p.atomic(p.optional(channel) + p.literal(" to=") + content + end)); @@ -94,58 +95,58 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & if (has_response_format) { auto response_format = p.rule("response-format", - p.literal("<|channel|>final") + constraint + p.literal("<|message|>") + + p.token("<|channel|>") + p.literal("final") + constraint + p.token("<|message|>") + p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema))); return p.zero_or_more(start + analysis) + start + response_format; } if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - auto tool_choice = p.choice(); + auto in_role = p.choice(); + auto in_channel = p.choice(); foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) { const auto & function = tool.at("function"); std::string name = function.at("name"); const auto params = common_chat_tool_parameters(function); - auto func_name = p.literal(" to=functions.") + p.tool_name(p.literal(name)); + auto func_name = p.tool_name(p.literal(name)); auto args = p.tool_args(p.schema(p.json(), "tool-" + std::to_string(tool_index) + "-schema", params)); // recipient in role header // <|start|>assistant to=functions.NAME<|channel|>(commentary|analysis)[constraint]<|message|>ARGS - auto tool_in_role = p.tool(p.tool_open(func_name + channel + constraint + p.literal("<|message|>")) + args); + in_role |= p.rule("tool-" + std::to_string(tool_index) + "-in-role", + p.atomic(func_name + channel + constraint + p.token("<|message|>")) + args); // recipient in channel header - // <|channel|>(commentary|analysis) to=functions.NAME[constraint]<|message|>ARGS - auto tool_in_channel = p.tool(p.tool_open(channel + func_name + constraint + p.literal("<|message|>")) + args); - - tool_choice |= p.rule("tool-" + std::to_string(tool_index), tool_in_role | tool_in_channel); + // <|start|>assistant<|channel|>(commentary|analysis) to=functions.NAME[constraint]<|message|>ARGS + in_channel |= p.rule("tool-" + std::to_string(tool_index) + "-in-channel", + p.atomic(func_name + constraint + p.token("<|message|>")) + args); }); - auto tool_call = p.trigger_rule("tool-call", tool_choice); + // The recipient prefix is the trigger, so the grammar constrains the name that follows it + auto to_functions = p.tool_open(p.literal(" to=functions.")); + auto tool_call = p.choice({ + p.trigger_rule("tool-call-in-role", start_assistant + to_functions, in_role), + p.trigger_rule("tool-call-in-channel", start_assistant + channel + to_functions, in_channel) + }); if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) { - return p.zero_or_more(start + any) + start + tool_call; + return p.zero_or_more(start + any) + tool_call; } - return p.zero_or_more(start + any) + start + (tool_call | final_msg); + return p.zero_or_more(start + any) + (tool_call | (start + final_msg)); } return p.zero_or_more(start + any) + start + (final_msg | unsolicited); }); if (include_grammar) { - data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); + // The grammar scans for the triggers itself, so the sampler runs it from the first token + data.grammar_lazy = false; data.grammar = build_grammar([&](const common_grammar_builder & builder) { - data.parser.build_grammar(builder, data.grammar_lazy); + data.parser.build_grammar(builder, !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO); }); - - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^\\s+to$" }, - { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^<\\|channel\\|>(?:commentary|analysis)\\s+to=functions$" }, - { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(\\s+to)" }, - { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(<\\|channel\\|>(?:commentary|analysis)\\s+to)" } - }; } return data; diff --git a/common/parsers/harmony-xml.cpp b/common/parsers/harmony-xml.cpp index bf05fac5d..1a7e3cb82 100644 --- a/common/parsers/harmony-xml.cpp +++ b/common/parsers/harmony-xml.cpp @@ -99,8 +99,11 @@ common_chat_params common_chat_params_init_harmony_xml(const common_chat_templat auto arg_close = p.tool_arg_close(p.gbnf( p.literal(PARAM_CLOSE) | (p.literal("\n") + p.peek(p.optional(p.literal("\n")) + next_open)), gbnf_format_literal("\n") + " " + gbnf_format_literal(PARAM_CLOSE.substr(1)) + "?")); - auto arg_string = p.rule("rfl-arg-string", - p.ac(p.tool_arg_string_value(p.until_one_of(string_ends)) + arg_close, PARAM_CLOSE)); + auto string_value = p.tool_arg_string_value(p.until_one_of(string_ends)); + auto arg_string = p.rule("rfl-arg-string", p.through(string_value, p.tool_arg_close(p.literal(PARAM_CLOSE)))); + // the parser also accepts a value with no closing tag, the grammar still requires the closing tag + auto missing_close = string_value + p.tool_arg_close(p.literal("\n") + p.peek(p.optional(p.literal("\n")) + next_open)); + arg_string = p.gbnf(arg_string | missing_close, "rfl-arg-string"); auto arg_sep = p.optional(p.literal("\n")); std::string call_close_gbnf = gbnf_format_literal("<|end|>"); diff --git a/common/parsers/k2-horizon.cpp b/common/parsers/k2-horizon.cpp index e688cd919..b02330d2d 100644 --- a/common/parsers/k2-horizon.cpp +++ b/common/parsers/k2-horizon.cpp @@ -91,7 +91,7 @@ common_chat_params common_chat_params_init_k2_horizon(const common_chat_template } auto think_body = p.until_one_of(data.thinking_end_tags); auto think_block = [&](const common_peg_parser & body) { - return p.optional(THINK_START + p.space() + p.ac(body + think_end, data.thinking_end_tags)); + return p.optional(THINK_START + p.space() + p.through(body, think_end)); }; auto reasoning = extract_reasoning ? think_block(p.reasoning(think_body)) : p.eps(); @@ -110,7 +110,7 @@ common_chat_params common_chat_params_init_k2_horizon(const common_chat_template tool_choice = p.standard_json_tools(CALL_START, CALL_END, inputs.tools, false, true); } else { auto arg_close = p.tool_arg_close(p.literal(ARG_VAL_END)); - auto arg_string = p.rule("xml-arg-string", p.ac(p.tool_arg_string_value(p.until(ARG_VAL_END)) + arg_close, ARG_VAL_END)); + auto arg_string = p.rule("xml-arg-string", p.through(p.tool_arg_string_value(p.until(ARG_VAL_END)), arg_close)); // The models leave out even when asked for xml_typed auto arg_type = call_format == "xml_typed" ? p.optional(ARG_TYPE + p.until(ARG_TYPE_END) + ARG_TYPE_END + p.space()) : p.eps(); diff --git a/common/parsers/minicpm5.cpp b/common/parsers/minicpm5.cpp index 7609865e0..ef0fed7fc 100644 --- a/common/parsers/minicpm5.cpp +++ b/common/parsers/minicpm5.cpp @@ -63,8 +63,8 @@ common_chat_params common_chat_params_init_minicpm5(const common_chat_template & // CDATA lets a value carry characters that would otherwise close the tag (e.g. // ); capture the inner text only, excluding the CDATA markers. auto string_value = p.choice({ - p.literal("")) + p.literal("]]>"), "]]>") + p.tool_arg_close(p.literal("")), - p.negate(p.literal("")) + p.tool_arg_close(p.literal("")), "") + p.literal("")), "]]>") + p.tool_arg_close(p.literal("")), + p.negate(p.literal("")), p.tool_arg_close(p.literal(""))) }); auto tool_choice = p.choice(); diff --git a/common/parsers/minimax-m3.cpp b/common/parsers/minimax-m3.cpp index 6080bfb00..dba02b238 100644 --- a/common/parsers/minimax-m3.cpp +++ b/common/parsers/minimax-m3.cpp @@ -64,8 +64,8 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template if (extract_reasoning) { auto block = inputs.enable_thinking ? p.literal(THINK_START) + p.space() + - p.ac(p.reasoning(p.until(THINK_END)) + p.literal(THINK_END), THINK_END) - : p.literal(THINK_START) + p.ac(p.until(THINK_END) + p.literal(THINK_END), THINK_END); + p.through(p.reasoning(p.until(THINK_END)), THINK_END) + : p.literal(THINK_START) + p.through(p.until(THINK_END), THINK_END); // A turn without reasoning is prefixed with a bare , written either by the // generation prompt (thinking_mode = "disabled") or by the model itself. @@ -113,7 +113,7 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template // A string accepts anything, so a union with a string alternative is a string if (schema.may_be_string()) { - return p.ac(p.tool_arg_string_value(p.until(close)) + close_tag, close); + return p.through(p.tool_arg_string_value(p.until(close)), close_tag); } if (schema.kind() == common_chat_schema::KIND_ANY_OF) { diff --git a/common/parsers/muse-glimmer.cpp b/common/parsers/muse-glimmer.cpp index 78ff50727..a1ee40f49 100644 --- a/common/parsers/muse-glimmer.cpp +++ b/common/parsers/muse-glimmer.cpp @@ -17,7 +17,7 @@ common_chat_params common_chat_params_init_muse_glimmer(const common_chat_templa data.supports_thinking = true; data.preserved_tokens = { - "<|start|>", "<|message|>", "<|eom|>", "<|eot|>", + "<|start|>", "<|message|>", "<|eom|>", // ATEM tool-call markup emitted on " to=" turns. "", "", "", "", @@ -48,37 +48,38 @@ common_chat_params common_chat_params_init_muse_glimmer(const common_chat_templa // Constrained grammar whenever tools are offered or a response format is requested. auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); - data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto start = p.rule("start", p.literal("<|start|>assistant")); + data.parser = build_chat_peg_parser(inputs.specials, [&](common_chat_peg_builder & p) { + auto start_token = p.token("<|start|>"); + auto message = p.token("<|message|>"); + auto eom = p.token("<|eom|>"); + auto start = p.rule("start", start_token + p.literal("assistant")); if (!extract_reasoning && !include_grammar) { return start + p.content(p.rest()); } if (extract_reasoning) { - p.rule("analysis", p.literal(" to=self<|message|>") + p.reasoning(p.until("<|eom|>")) + p.literal("<|eom|>")); + p.rule("analysis", p.literal(" to=self") + message + p.reasoning(p.until(eom)) + eom); } else { - p.rule("analysis", p.literal(" to=self<|message|>") + p.content(p.until("<|eom|>")) + p.literal("<|eom|>")); + p.rule("analysis", p.literal(" to=self") + message + p.content(p.until(eom)) + eom); } auto analysis = p.ref("analysis"); auto recipient = p.optional(p.literal(" to=user")); - auto final_msg = p.rule("final", recipient + p.literal("<|message|>") + - p.content(p.until_one_of({ "<|eot|>", "<|eom|>" }))); + auto final_msg = p.rule("final", recipient + message + p.content(p.until(eom))); if (has_response_format) { auto response_json = p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)); auto response_format = p.rule("response-format", - recipient + p.literal("<|message|>") + + recipient + message + ((p.literal("```json") + p.space() + response_json + p.space() + p.literal("```")) | response_json)); return p.zero_or_more(start + analysis) + start + response_format; } if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - auto string_value = p.ac( - p.tool_arg_string_value(p.until("")) + p.tool_arg_close(p.literal("")), - ""); + auto string_value = p.through(p.tool_arg_string_value(p.until("")), + p.tool_arg_close(p.literal(""))); auto tool_choice = p.choice(); foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) { @@ -106,40 +107,35 @@ common_chat_params common_chat_params_init_muse_glimmer(const common_chat_templa args = p.zero_or_more(p.choice(arg_rules) + p.space()); } - auto tool_parser = p.tool( - p.tool_open(p.literal(" to=") + p.until("<|message|>") + - p.literal("<|message|>") + p.space() + - p.literal("") + p.space()) + auto open = p.tool_open(start_token + p.literal("assistant to=") + p.tool_name(p.literal(name)) + message); + auto body = p.rule("tool-" + std::to_string(tool_index), + p.literal("") + p.space() + p.literal("") + p.space() << p.tool_args(args) << p.tool_close(p.literal("") + p.space() + p.literal(""))); - tool_choice |= p.rule("tool-" + std::to_string(tool_index), tool_parser); + tool_choice |= open + body; }); - auto tool_calls = inputs.parallel_tool_calls - ? p.trigger_rule("tool-call", tool_choice + p.zero_or_more(p.literal("<|eom|>") + start + tool_choice)) - : p.trigger_rule("tool-call", tool_choice); - + auto more = inputs.parallel_tool_calls ? p.zero_or_more(eom + tool_choice) : p.eps(); + auto tool_calls = p.rule("tool-calls", tool_choice + more); if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) { - return p.zero_or_more(start + analysis) + start + tool_calls; + return p.zero_or_more(start + analysis) + tool_calls; } - auto trailing_calls = p.optional(p.literal("<|eom|>") + start + tool_calls); - return p.zero_or_more(start + analysis) + start + (tool_calls | (final_msg + trailing_calls)); + auto trailing_calls = p.optional(eom + tool_calls); + return p.zero_or_more(start + analysis) + (tool_calls | (start + final_msg + trailing_calls)); } return p.zero_or_more(start + analysis) + start + final_msg; }); if (include_grammar) { - data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); + // Every message starts with a recipient header, so the grammar is not lazy. + // The model cannot generate a recipient that is not self, user, or a tool. + data.grammar_lazy = false; data.grammar = build_grammar([&](const common_grammar_builder & builder) { - data.parser.build_grammar(builder, data.grammar_lazy); + data.parser.build_grammar(builder); }); - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, - "(?:^|<\\|start\\|>assistant)( to=(?!self<\\|message\\|>)(?!user<\\|message\\|>)[^<]*?<\\|message\\|>)" }, - }; } return data; diff --git a/common/parsers/muse-spark.cpp b/common/parsers/muse-spark.cpp index 945ee2aaf..74a393f4e 100644 --- a/common/parsers/muse-spark.cpp +++ b/common/parsers/muse-spark.cpp @@ -86,9 +86,8 @@ common_chat_params common_chat_params_init_muse_spark(const common_chat_template } if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - auto string_value = p.ac( - p.tool_arg_string_value(p.until("")) + p.tool_arg_close(p.literal("")), - ""); + auto string_value = p.through(p.tool_arg_string_value(p.until("")), + p.tool_arg_close(p.literal(""))); auto tool_choice = p.choice(); foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) { diff --git a/common/parsers/qwen3-coder.cpp b/common/parsers/qwen3-coder.cpp index b8dc02267..c943b93f3 100644 --- a/common/parsers/qwen3-coder.cpp +++ b/common/parsers/qwen3-coder.cpp @@ -59,27 +59,16 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat data.prompt.append(data.generation_prompt); } - std::vector tool_call_starts = { "" }; - - if (is_qwen3_coder) { - // Match complete opener for Qwen3-Coder models that occasionally omit the - // starting . The model may hallucinate a tool name, but it is preferable over - // constraining on - foreach_function(inputs.tools, [&](size_t, const json & tool) { - const std::string name = tool.at("function").at("name"); - tool_call_starts.push_back(""); - }); - } - - data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + data.parser = build_chat_peg_parser(inputs.specials, [&](common_chat_peg_builder & p) { auto generation_prompt = p.literal(GEN_PREFIX); // The template strips the newline before , so leave it out of the reasoning - auto think_body = p.until_one_of({ "\n", "", "" }); - auto think_close = p.literal("\n") | p.literal("") | p.peek(p.literal("")); + auto think_end = (p.literal("\n") + p.token("")) | p.token(""); + auto think_body = p.until(think_end | p.token("")); + auto think_close = think_end | p.peek(p.token("")); - auto reasoning_extracted = p.optional("" + p.space() + p.reasoning(think_body) + think_close); - auto reasoning_skipped = p.optional("" + p.space() + think_body + think_close); + auto reasoning_extracted = p.optional(p.token("") + p.space() + p.reasoning(think_body) + think_close); + auto reasoning_skipped = p.optional(p.token("") + p.space() + think_body + think_close); auto reasoning = p.eps(); if (supports_reasoning && extract_reasoning) { @@ -99,7 +88,13 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { auto arg_close = p.tool_arg_close(p.literal("\n\n")); auto arg_string = p.rule("xml-arg-string", - p.ac(p.tool_arg_string_value(p.until("\n\n")) + arg_close, "\n\n")); + p.through(p.tool_arg_string_value(p.until("\n\n")), arg_close)); + + auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; + + auto more = inputs.parallel_tool_calls ? p.zero_or_more(p.ref("tool-call")) : p.eps(); + auto starts = p.token(""); + auto tool_calls = p.choice(); auto tool_choice = p.choice(); foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) { @@ -155,35 +150,29 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat args = args + p.zero_or_more(p.choice(optional_args)); } - auto func = p.tool(p.tool_open("\n") + - p.tool_args(args) + - p.tool_close(p.literal("\n"))); + auto opener = p.tool_open(""); + auto body = p.literal("\n") + p.tool_args(args) + p.tool_close(p.literal("\n")); - tool_choice |= p.rule("tool-" + std::to_string(tool_index), func); + tool_choice |= p.rule("tool-" + std::to_string(tool_index), p.tool(opener + body)); + + if (is_qwen3_coder) { + // Qwen3-Coder models sometimes omit , so the full opener is also a trigger. + // Do not trigger on . + tool_calls |= p.trigger_rule("tool-calls-" + std::to_string(tool_index), opener, body + p.token("") + p.space() + more); + starts |= opener; + } }); - auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; + auto tool_call_body = tool_choice + p.token("") + p.space(); + p.rule("tool-call", p.token("") + p.literal("\n") + tool_call_body); - auto tool_call_body = tool_choice + "" + p.space(); - auto tool_call = p.rule("tool-call", "\n" + tool_call_body); - - // Qwen3-Coder models may occasionally omit the token. - auto tool_call_first = is_qwen3_coder ? - p.rule("tool-call-first", p.optional(p.literal("\n")) + tool_call_body) : - tool_call; - - auto calls = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first; - auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1)); + tool_calls |= p.trigger_rule("tool-calls", p.token(""), p.literal("\n") + tool_call_body + more); // The templates separate content from the tool calls with one or two newlines, // stop the content before them - std::vector content_ends; - for (const auto & start : tool_call_starts) { - content_ends.insert(content_ends.end(), { "\n\n" + start, "\n" + start, start }); - } + auto content_end = (p.literal("\n\n") + starts) | (p.literal("\n") + starts) | starts; - return generation_prompt + - (reasoning << p.content(p.until_one_of(content_ends)) << tool_calls); + return generation_prompt + (reasoning << p.content(p.until(content_end)) << p.repeat(tool_calls, min_calls, 1)); } // Content only parser @@ -191,17 +180,11 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat }); if (include_grammar) { - data.grammar_lazy = !has_response_format && has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; - + // The grammar scans for the triggers itself, so the sampler runs it from the first token + data.grammar_lazy = false; data.grammar = build_grammar([&](const common_grammar_builder & builder) { - data.parser.build_grammar(builder, data.grammar_lazy); + data.parser.build_grammar(builder, !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO); }); - - if (data.grammar_lazy) { - for (const auto & start : tool_call_starts) { - data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, start }); - } - } } return data; diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index d980cfc76..924289097 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -208,6 +208,89 @@ void common_peg_ast_arena::visit(const common_peg_parse_result & result, const c } } +common_peg_tokens::common_peg_tokens(const llama_vocab * vocab) { + const llama_token n_tokens = llama_vocab_n_tokens(vocab); + for (llama_token id = 0; id < n_tokens; ++id) { + const auto attr = llama_vocab_get_attr(vocab, id); + if (attr & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED)) { + std::string text = llama_vocab_get_text(vocab, id); + ids.emplace(text, id); + tokens.emplace(id, common_peg_token{ std::move(text), attr }); + } + } +} + +llama_token common_peg_tokens::token_id(const std::string & text) const { + auto it = ids.find(text); + return it != ids.end() ? it->second : LLAMA_TOKEN_NULL; +} + +// Insert every delimiter the parser matches into the trie, starting from each node in from. Returns the end nodes. +// A choice adds the delimiters of each branch. A sequence joins every combination of its children. +static std::set insert_delimiter(common_trie & trie, const common_peg_arena & arena, common_peg_parser_id id, + std::set from = { 0 }) { + return std::visit([&](const auto & p) -> std::set { + using T = std::decay_t; + if constexpr (std::is_same_v || std::is_same_v) { + return insert_delimiter(trie, arena, p.child, std::move(from)); + } else if constexpr (std::is_same_v) { + std::set ends; + for (size_t node : from) { + size_t pos = 0; + while (pos < p.literal.size()) { + auto result = common_parse_utf8_codepoint(p.literal, pos); + if (result.status != utf8_parse_result::SUCCESS) { + break; + } + node = trie.extend(node, common_trie::symbol::codepoint(result.codepoint)); + pos += result.bytes_consumed; + } + ends.insert(node); + } + return ends; + } else if constexpr (std::is_same_v) { + std::set ends; + for (size_t node : from) { + ends.insert(trie.extend(node, common_trie::symbol::token(p.token))); + } + return ends; + } else if constexpr (std::is_same_v) { + std::set ends; + for (auto child : p.children) { + ends.merge(insert_delimiter(trie, arena, child, from)); + } + return ends; + } else if constexpr (std::is_same_v) { + for (auto child : p.children) { + from = insert_delimiter(trie, arena, child, std::move(from)); + } + return from; + } else { + throw std::invalid_argument("a delimiter may only be a sequence of literals, tokens, and choices of them, possibly wrapped in tags and atomics"); + } + }, arena.get(id)); +} + +// Build the matcher of an until parser, each pattern valued by the branch it belongs to +static common_trie build_until_matcher(const common_peg_arena & arena, + const std::vector & branches, + bool include) { + common_trie matcher; + for (size_t b = 0; b < branches.size(); b++) { + for (size_t end : insert_delimiter(matcher, arena, branches[b].delimiter)) { + if (end == 0) { + throw std::invalid_argument("delimiter must not match the empty string"); + } + // nothing follows a delimiter without include, so the first branch to claim it wins + auto owner = matcher.mark(end, (int32_t) b); + if (owner != (int32_t) b && include) { + throw std::invalid_argument("until branches " + std::to_string(owner) + " and " + std::to_string(b) + " must not share a delimiter"); + } + } + } + return matcher; +} + struct parser_executor; common_peg_parser_id common_peg_arena::add_parser(common_peg_parser_variant parser) { @@ -298,6 +381,49 @@ struct parser_executor { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); } + common_peg_parse_result operator()(const common_peg_token_parser & p) { + if (start_pos >= ctx.input.size()) { + if (!ctx.is_lenient()) { + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); + } + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos); + } + + if (start_pos >= ctx.tokens.size() || ctx.tokens[start_pos] != p.token) { + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); + } + + // Skip the rest of the piece, up to the next token or the end of the input + auto pos = start_pos + 1; + while (pos < ctx.input.size() && ctx.tokens[pos] == LLAMA_TOKEN_NULL) { + ++pos; + } + + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); + } + + common_peg_parse_result operator()(const common_peg_token_not_parser & p) { + if (start_pos >= ctx.input.size()) { + if (!ctx.is_lenient()) { + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); + } + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos); + } + + if (start_pos >= ctx.tokens.size() || ctx.tokens[start_pos] == LLAMA_TOKEN_NULL || + std::find(p.tokens.begin(), p.tokens.end(), ctx.tokens[start_pos]) != p.tokens.end()) { + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); + } + + // Skip the rest of the piece, up to the next token or the end of the input + auto pos = start_pos + 1; + while (pos < ctx.input.size() && ctx.tokens[pos] == LLAMA_TOKEN_NULL) { + ++pos; + } + + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); + } + common_peg_parse_result operator()(const common_peg_sequence_parser & p) { if (ctx.is_debug()) { LOG_DBG("%sSEQ start at %zu '%s' (%zu children)\n", debug_indent().c_str(), start_pos, @@ -689,7 +815,11 @@ struct parser_executor { } common_peg_parse_result operator()(const common_peg_until_parser & p) const { - common_trie matcher(p.delimiters); + if (p.include) { + return parse_through(p); + } + + const auto & matcher = p.matcher; // Scan input and check for delimiters size_t pos = start_pos; @@ -697,24 +827,24 @@ struct parser_executor { std::vector invalid_utf8; while (pos < ctx.input.size()) { - auto utf8_result = common_parse_utf8_codepoint(ctx.input, pos); + auto step = matcher.next(ctx.input, ctx.tokens, pos); - if (utf8_result.status == utf8_parse_result::INCOMPLETE && ctx.is_lenient()) { + if (step.status == utf8_parse_result::INCOMPLETE && ctx.is_lenient()) { // The rest of the sequence may still arrive, return what we have so far return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, last_valid_pos, {}, std::move(invalid_utf8)); } - if (utf8_result.status != utf8_parse_result::SUCCESS) { + if (step.status != utf8_parse_result::SUCCESS) { // Malformed UTF-8, or a sequence truncated by the end of a complete input. // A delimiter cannot start inside bytes that fail to decode, so consume them and move on - invalid_utf8.push_back({pos, utf8_result.bytes_consumed}); - pos += utf8_result.bytes_consumed; + invalid_utf8.push_back({pos, step.bytes_consumed}); + pos += step.bytes_consumed; last_valid_pos = pos; continue; } // Check if a delimiter starts at this position - auto match = matcher.check_at(ctx.input, pos); + auto match = matcher.check_at(ctx.input, ctx.tokens, pos); if (match == common_trie::COMPLETE_MATCH) { // Found a complete delimiter, return everything before it @@ -727,7 +857,7 @@ struct parser_executor { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, {}, std::move(invalid_utf8)); } - pos += utf8_result.bytes_consumed; + pos += step.bytes_consumed; last_valid_pos = pos; } @@ -738,6 +868,48 @@ struct parser_executor { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, last_valid_pos, {}, std::move(invalid_utf8)); } + // Parse content, then the first branch whose delimiter and rest follow it, like C (D1 R1 | D2 R2 | ...) + common_peg_parse_result parse_through(const common_peg_until_parser & p) const { + auto content = arena.parse(p.content, ctx, start_pos); + if (content.fail() || content.need_more_input()) { + return content; + } + + auto pos = content.end; + for (const auto & branch : p.branches) { + auto nodes = content.nodes; + auto invalid_utf8 = content.invalid_utf8; + auto end = pos; + bool matched = true; + + for (auto id : { branch.delimiter, branch.rest }) { + if (id == COMMON_PEG_INVALID_PARSER_ID) { + continue; + } + auto result = arena.parse(id, ctx, end); + if (result.fail()) { + matched = false; + break; + } + nodes.insert(nodes.end(), result.nodes.begin(), result.nodes.end()); + invalid_utf8.insert(invalid_utf8.end(), result.invalid_utf8.begin(), result.invalid_utf8.end()); + end = result.end; + if (result.need_more_input()) { + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, end, std::move(nodes), std::move(invalid_utf8)); + } + } + if (matched) { + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, end, std::move(nodes), std::move(invalid_utf8)); + } + } + + if (p.optional && pos == ctx.input.size()) { + auto type = ctx.is_lenient() ? COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT : COMMON_PEG_PARSE_RESULT_SUCCESS; + return common_peg_parse_result(type, start_pos, pos, std::move(content.nodes), std::move(content.invalid_utf8)); + } + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos, pos); + } + common_peg_parse_result operator()(const common_peg_schema_parser & p) { return arena.parse(p.child, ctx, start_pos); } @@ -823,10 +995,6 @@ struct parser_executor { common_peg_parse_result operator()(const common_peg_gbnf_parser & p) { return arena.parse(p.child, ctx, start_pos); } - - common_peg_parse_result operator()(const common_peg_ac_parser & p) { - return arena.parse(p.child, ctx, start_pos); - } }; common_peg_parse_result common_peg_arena::parse(common_peg_parse_context & ctx, size_t start) const { @@ -899,19 +1067,33 @@ void common_peg_arena::resolve_refs() { std::is_same_v || std::is_same_v || std::is_same_v || - std::is_same_v || - std::is_same_v) { + std::is_same_v) { p.child = resolve_ref(p.child); + } else if constexpr (std::is_same_v) { + if (p.content != COMMON_PEG_INVALID_PARSER_ID) { + p.content = resolve_ref(p.content); + } + for (auto & branch : p.branches) { + branch.delimiter = resolve_ref(branch.delimiter); + if (branch.rest != COMMON_PEG_INVALID_PARSER_ID) { + branch.rest = resolve_ref(branch.rest); + } + } } else if constexpr (std::is_same_v) { p.child = resolve_ref(p.child); + if (p.start != COMMON_PEG_INVALID_PARSER_ID) { + p.start = resolve_ref(p.start); + p.rest = resolve_ref(p.rest); + } } else if constexpr (std::is_same_v) { p.child = resolve_ref(p.child); } else if constexpr (std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || - std::is_same_v || std::is_same_v || + std::is_same_v || + std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || @@ -955,6 +1137,14 @@ std::string common_peg_arena::dump_impl(common_peg_parser_id return "End"; } else if constexpr (std::is_same_v) { return "Literal(" + p.literal + ")"; + } else if constexpr (std::is_same_v) { + return "Token(" + p.piece + ", " + std::to_string(p.token) + ")"; + } else if constexpr (std::is_same_v) { + std::vector parts; + for (auto token : p.tokens) { + parts.push_back(std::to_string(token)); + } + return "TokenNot(" + string_join(parts, ", ") + ")"; } else if constexpr (std::is_same_v) { std::vector parts; for (const auto & child : p.children) { @@ -981,8 +1171,6 @@ std::string common_peg_arena::dump_impl(common_peg_parser_id return "Atomic(" + dump_impl(p.child, visited) + ")"; } else if constexpr (std::is_same_v) { return "Gbnf(" + p.grammar + ", " + dump_impl(p.child, visited) + ")"; - } else if constexpr (std::is_same_v) { - return "Ac(" + string_join(p.delimiters, " | ") + ", " + dump_impl(p.child, visited) + ")"; } else if constexpr (std::is_same_v) { return "Any"; } else if constexpr (std::is_same_v) { @@ -995,7 +1183,18 @@ std::string common_peg_arena::dump_impl(common_peg_parser_id } else if constexpr (std::is_same_v) { return "String(" + std::string(1, p.delimiter) + ")"; } else if constexpr (std::is_same_v) { - return "Until(" + string_join(p.delimiters, " | ") + ")"; + std::vector branches; + for (const auto & branch : p.branches) { + auto text = dump_impl(branch.delimiter, visited); + if (branch.rest != COMMON_PEG_INVALID_PARSER_ID) { + text += " " + dump_impl(branch.rest, visited); + } + branches.push_back(std::move(text)); + } + if (p.include) { + return "Until(" + dump_impl(p.content, visited) + ", " + string_join(branches, " | ") + (p.optional ? ", optional" : "") + ")"; + } + return "Until(" + string_join(branches, " | ") + ")"; } else if constexpr (std::is_same_v) { return "Schema(" + dump_impl(p.child, visited) + ", " + (p.node ? common_chat_schema::kind_name(p.node->kind()) : "null") + ")"; } else if constexpr (std::is_same_v) { @@ -1089,6 +1288,67 @@ common_peg_parser operator|(const std::string & str, const common_peg_parser & p common_peg_parser_builder::common_peg_parser_builder() {} +common_peg_parser_builder::common_peg_parser_builder(common_peg_tokens tokens) { + arena_.tokens_ = std::move(tokens); +} + +common_peg_parser common_peg_parser_builder::until_one_of(const std::vector & delimiters) { + common_peg_until_parser p; + for (const auto & d : delimiters) { + p.branches.push_back({ literal(d).id() }); + } + p.matcher = build_until_matcher(arena_, p.branches, p.include); + return add(std::move(p)); +} + +common_peg_parser common_peg_parser_builder::until(const common_peg_parser & delimiter) { + common_peg_until_parser p; + p.branches = { { delimiter.id() } }; + p.matcher = build_until_matcher(arena_, p.branches, p.include); + return add(std::move(p)); +} + +common_peg_parser common_peg_parser_builder::through(const common_peg_parser & content, const std::vector & branches, bool optional) { + if (branches.empty()) { + throw std::runtime_error("through requires at least one branch"); + } + common_peg_until_parser p; + p.branches = branches; + p.include = true; + p.optional = optional; + p.content = content.id(); + p.matcher = build_until_matcher(arena_, p.branches, p.include); + return add(std::move(p)); +} + +common_peg_parser common_peg_parser_builder::token(const std::string & piece) { + auto token = arena_.tokens_.token_id(piece); + if (token == LLAMA_TOKEN_NULL) { + return literal(piece); + } + return add(common_peg_token_parser{token, piece}); +} + +common_peg_parser common_peg_parser_builder::token(const std::vector & pieces, bool negate) { + if (!negate) { + std::vector alts; + for (const auto & piece : pieces) { + alts.push_back(token(piece)); + } + return choice(alts); + } + + common_peg_token_not_parser p; + for (const auto & piece : pieces) { + auto token = arena_.tokens_.token_id(piece); + if (token == LLAMA_TOKEN_NULL) { + throw std::invalid_argument("token is not registered with the builder: " + piece); + } + p.tokens.push_back(token); + } + return add(p); +} + common_peg_parser common_peg_parser_builder::sequence(const std::vector & parsers) { // Flatten nested sequences std::vector flattened; @@ -1195,6 +1455,17 @@ common_peg_parser common_peg_parser_builder::rule(const std::string & name, cons return ref(clean_name); } +common_peg_parser common_peg_parser_builder::trigger_rule(const std::string & name, const common_peg_parser & start, const common_peg_parser & rest) { + common_trie scratch; + if (insert_delimiter(scratch, arena_, start.id()).count(0)) { + throw std::invalid_argument("trigger start must not match the empty string"); + } + auto clean_name = gbnf_format_rule_name(name); + auto rule_id = arena_.add_parser(common_peg_rule_parser{clean_name, (start + rest).id(), true, start.id(), rest.id()}); + arena_.add_rule(clean_name, rule_id); + return ref(clean_name); +} + void common_peg_parser_builder::set_root(const common_peg_parser & p) { arena_.set_root(p.id()); } @@ -1392,13 +1663,6 @@ common_peg_parser common_peg_parser_builder::json_member(const std::string & key }); } -common_peg_parser common_peg_parser_builder::ac(const common_peg_parser & p, const std::vector & delimiters) { - if (delimiters.empty()) { - throw std::runtime_error("ac parser requires at least one delimiter"); - } - return add(common_peg_ac_parser{p, delimiters}); -} - static std::string gbnf_escape_char_class(uint32_t c) { if (c == '-' || c == ']' || c == '[' || c == '\\') { return "\\" + std::string(1, (char) c); @@ -1457,15 +1721,63 @@ static std::string gbnf_char_class(const std::vector & chars, bool neg return s + "]"; } +// Token set such as <[1,5-9]>, with consecutive ids collapsed into ranges +static std::string gbnf_token_set(const std::vector & tokens, bool negate) { + std::string s = negate ? "!<[" : "<["; + for (size_t i = 0; i < tokens.size();) { + size_t j = i; + while (j + 1 < tokens.size() && tokens[j + 1] == tokens[j] + 1) { + j++; + } + s += (i == 0 ? "" : ",") + std::to_string(tokens[i]); + if (j > i) { + s += "-" + std::to_string(tokens[j]); + } + i = j + 1; + } + return s + "]>"; +} + +// Symbols on the transitions out of an automaton state, codepoints and tokens kept apart +struct gbnf_ac_symbols { + std::vector chars; + std::vector tokens; + + void add(const common_trie::symbol & sym) { + (sym.is_token() ? tokens : chars).push_back(sym.value); + } + + // One alternative for the codepoints and one for the tokens, each followed by next if given + std::vector alternatives(const std::string & next = "") const { + std::vector alts; + const std::string tail = next.empty() ? "" : " " + next; + if (!chars.empty()) { + alts.push_back(gbnf_char_class(chars, false) + tail); + } + if (!tokens.empty()) { + alts.push_back(gbnf_token_set(tokens, false) + tail); + } + return alts; + } +}; + static std::string gbnf_ac_grammar( - const common_grammar_builder & builder, - const std::string & prefix, - const std::vector & strings, - const std::function &, - const std::map> &, - const std::vector &, + const common_grammar_builder & builder, + const std::string & prefix, + const common_trie & trie, + const std::set & opaque, + const std::function &, + const std::map &, + const std::string &, const std::function &)> & build_rule) { - common_aho_corasick ac(strings); + common_aho_corasick ac(trie); + + // With only tokens in the alphabet, the grammar can step a whole token at a time + bool token_only = true; + for (const auto & sym : ac.alphabet) { + token_only = token_only && sym.is_token(); + } auto state_name = [&](size_t s) -> std::string { if (s == 0) { @@ -1481,27 +1793,43 @@ static std::string gbnf_ac_grammar( continue; // match states } - std::map> buckets; - std::vector completing; // chars that complete a delimiter - std::vector specific; // chars with an explicit transition - for (uint32_t c : ac.alphabet) { - size_t d = ac.next(q, c); + std::map buckets; + std::map completing_to; // completing symbols by the match state they reach + gbnf_ac_symbols completing; // symbols that complete a delimiter + gbnf_ac_symbols specific; // symbols with an explicit transition + for (const auto & sym : ac.alphabet) { + size_t d = ac.next(q, sym); if (ac.is_terminal(d)) { - completing.push_back(c); - specific.push_back(c); + completing.add(sym); + completing_to[d].add(sym); + specific.add(sym); } else if (d != 0) { - buckets[d].push_back(c); // specific non-root destination - specific.push_back(c); + buckets[d].add(sym); // specific non-root destination + specific.add(sym); } } - builder.add_rule(state_name(q), build_rule(completing, buckets, specific, state_name)); - } + // every other symbol goes back to the start + std::string fallback; + if (token_only) { + fallback = gbnf_token_set(specific.tokens, true); + } else { + fallback = specific.chars.empty() ? "." : gbnf_char_class(specific.chars, true); - // An empty delimiter makes the start state terminal. Emit an entry rule - // that matches the empty string so the returned reference stays valid. - if (ac.is_terminal(0)) { - builder.add_rule(prefix, "|"); + // Char rules never match a token named by a token rule, so allow the other named tokens by name + std::vector others; + for (auto token : opaque) { + if (std::find(specific.tokens.begin(), specific.tokens.end(), token) == specific.tokens.end()) { + others.push_back(token); + } + } + if (!others.empty()) { + fallback = "(" + fallback + " | " + gbnf_token_set(others, false) + ")"; + } + } + fallback += " " + state_name(0); + + builder.add_rule(state_name(q), build_rule(completing, completing_to, buckets, fallback, state_name)); } return state_name(0); @@ -1514,52 +1842,104 @@ static std::string gbnf_ac_grammar( // ref: https://github.com/ggml-org/llama.cpp/pull/24839 static std::string gbnf_excluding_grammar(const common_grammar_builder & builder, const std::string & prefix, - const std::vector & strings) { - return gbnf_ac_grammar(builder, prefix, strings, - [](const std::vector & /*completing*/, - const std::map> & buckets, - const std::vector & specific, + const common_trie & trie, + const std::set & opaque) { + return gbnf_ac_grammar(builder, prefix, trie, opaque, + [](const gbnf_ac_symbols & /*completing*/, + const std::map & /*completing_to*/, + const std::map & buckets, + const std::string & fallback, const std::function & state_name) { - // every state is accepting and completing chars get no - // alternative, so a forbidden string can never be matched + // every state accepts and completing symbols get no alternative, so a forbidden string never matches std::string rhs = "|"; - for (const auto & [d, chars] : buckets) { - rhs += " " + gbnf_char_class(chars, false) + " " + state_name(d) + " |"; + for (const auto & [d, symbols] : buckets) { + for (const auto & alt : symbols.alternatives(state_name(d))) { + rhs += " " + alt + " |"; + } } - rhs += " " + gbnf_char_class(specific, true) + " " + state_name(0); + rhs += " " + fallback; return rhs; }); } -// GBNF grammar matching everything up to and including the first occurrence of -// any string in `strings`. Emits the Aho-Corasick automaton DFA and returns -// the start state rule name. -static std::string gbnf_including_grammar(const common_grammar_builder & builder, - const std::string & prefix, - const std::vector & strings) { - return gbnf_ac_grammar(builder, prefix, strings, - [](const std::vector & completing, - const std::map> & buckets, - const std::vector & specific, - const std::function & state_name) { +// GBNF grammar matching up to the first pattern in the trie, then the rest rule of that pattern's value. +// For the pattern "ab" with rest "x": +// +// prefix ::= [a] prefix-01 | [^a] prefix +// prefix-01 ::= [b] rest | [a] prefix-01 | [^ab] prefix +// rest ::= "x" +static std::string gbnf_including_grammar(const common_grammar_builder & builder, + const std::string & prefix, + const common_trie & trie, + const std::set & opaque, + const std::vector & rests, + bool optional) { + common_aho_corasick ac(trie); + + // every pattern ending at a match state, the longest and the ones reached by suffix links + auto rests_at = [&](size_t s, bool & terminal) { + std::vector alts; + terminal = false; + for (; s != 0; s = ac.fail[s]) { + auto pattern = ac.t.nodes[s].pattern; + if (pattern < 0) { + continue; + } + if (rests[pattern].empty()) { + terminal = true; + } else if (std::find(alts.begin(), alts.end(), rests[pattern]) == alts.end()) { + alts.push_back(rests[pattern]); + } + } + return alts; + }; + + return gbnf_ac_grammar(builder, prefix, trie, opaque, + [&](const gbnf_ac_symbols & /*completing*/, + const std::map & completing_to, + const std::map & buckets, + const std::string & fallback, + const std::function & state_name) { std::vector alts; - if (!completing.empty()) { - alts.push_back(gbnf_char_class(completing, false)); // terminate on match + gbnf_ac_symbols terminating; // symbols that complete a pattern with no rest + for (const auto & [d, symbols] : completing_to) { + bool terminal = false; + auto rest = rests_at(d, terminal); + if (!rest.empty()) { + for (auto & alt : symbols.alternatives(rest.size() == 1 ? rest[0] : "(" + string_join(rest, " | ") + ")")) { + alts.push_back(std::move(alt)); + } + } + if (terminal) { + terminating.chars.insert(terminating.chars.end(), symbols.chars.begin(), symbols.chars.end()); + terminating.tokens.insert(terminating.tokens.end(), symbols.tokens.begin(), symbols.tokens.end()); + } } - for (const auto & [d, chars] : buckets) { - alts.push_back(gbnf_char_class(chars, false) + " " + state_name(d)); + for (auto & alt : terminating.alternatives()) { + alts.push_back(std::move(alt)); } - // every other character keeps scanning from the start state - alts.push_back(gbnf_char_class(specific, true) + " " + state_name(0)); - return string_join(alts, " | "); + for (const auto & [d, symbols] : buckets) { + for (auto & alt : symbols.alternatives(state_name(d))) { + alts.push_back(std::move(alt)); + } + } + // every other symbol keeps scanning from the start state + alts.push_back(fallback); + return (optional ? "| " : "") + string_join(alts, " | "); }); } -static std::set collect_reachable_rules( +// The rules and the tokens named by token rules that the grammar of a parser reaches +struct common_peg_reachable { + std::set rules; + std::set tokens; +}; + +static void collect_reachable( const common_peg_arena & arena, - const common_peg_parser_id & rule + const common_peg_parser_id & root, + common_peg_reachable & reachable ) { - std::set reachable; std::set visited; std::function visit = [&](common_peg_parser_id id) { @@ -1571,13 +1951,27 @@ static std::set collect_reachable_rules( if constexpr (std::is_same_v || std::is_same_v || std::is_same_v || - std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v) { // These parsers do not have any children + } else if constexpr (std::is_same_v) { + reachable.tokens.insert(p.token); + } else if constexpr (std::is_same_v) { + reachable.tokens.insert(p.tokens.begin(), p.tokens.end()); + } else if constexpr (std::is_same_v) { + // The delimiters name their tokens in the grammar + if (p.content != COMMON_PEG_INVALID_PARSER_ID) { + visit(p.content); + } + for (const auto & branch : p.branches) { + visit(branch.delimiter); + if (branch.rest != COMMON_PEG_INVALID_PARSER_ID) { + visit(branch.rest); + } + } } else if constexpr (std::is_same_v) { for (auto child : p.children) { visit(child); @@ -1592,13 +1986,12 @@ static std::set collect_reachable_rules( std::is_same_v || std::is_same_v || std::is_same_v || - std::is_same_v || std::is_same_v) { visit(p.child); } else if constexpr (std::is_same_v) { if (visited.find(p.name) == visited.end()) { visited.insert(p.name); - reachable.insert(p.name); + reachable.rules.insert(p.name); visit(p.child); } } else if constexpr (std::is_same_v) { @@ -1611,12 +2004,13 @@ static std::set collect_reachable_rules( }, parser); }; - visit(rule); - return reachable; + visit(root); } // GBNF generation implementation void common_peg_arena::build_grammar(const common_grammar_builder & builder, bool lazy) const { + common_peg_reachable reachable; + // A raw string value is parsed by the child rather than constrained by the schema auto schema_delegates = [](const common_peg_schema_parser & s) -> bool { return !s.node || (s.raw && s.node->may_be_string()); @@ -1655,6 +2049,10 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo return ""; } else if constexpr (std::is_same_v) { return gbnf_format_literal(p.literal); + } else if constexpr (std::is_same_v) { + return "<[" + std::to_string(p.token) + "]>"; + } else if constexpr (std::is_same_v) { + return gbnf_token_set(std::vector(p.tokens.begin(), p.tokens.end()), true); } else if constexpr (std::is_same_v) { std::string s; for (const auto & child : p.children) { @@ -1746,10 +2144,19 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo const std::string delim(1, p.delimiter); return R"(( [^)" + delim + R"(\\] | "\\" ( [)" + delim + R"(\\/ bfnrt] | "u" [0-9a-fA-F]{4} ) )*)"; } else if constexpr (std::is_same_v) { - if (p.delimiters.empty()) { - return ".*"; + auto prefix = "until-" + std::to_string(id); + if (!p.include) { + if (p.branches.empty()) { + return ".*"; + } + return gbnf_excluding_grammar(builder, prefix, p.matcher, reachable.tokens); } - return gbnf_excluding_grammar(builder, "until-" + std::to_string(id), p.delimiters); + std::vector branch_rests; + for (size_t b = 0; b < p.branches.size(); b++) { + auto rest = p.branches[b].rest; + branch_rests.push_back(rest == COMMON_PEG_INVALID_PARSER_ID ? "" : builder.add_rule(prefix + "-rest-" + std::to_string(b), to_gbnf(rest))); + } + return gbnf_including_grammar(builder, prefix, p.matcher, reachable.tokens, branch_rests, p.optional); } else if constexpr (std::is_same_v) { if (schema_delegates(p)) { return to_gbnf(p.child); @@ -1766,38 +2173,52 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo return to_gbnf(p.child); } else if constexpr (std::is_same_v) { return p.grammar; - } else if constexpr (std::is_same_v) { - return gbnf_including_grammar(builder, "ac-" + std::to_string(id), p.delimiters); } else { static_assert(is_always_false_v); } }, parser); }; - // Collect reachable rules - std::set reachable_rules; - - if (lazy) { - // Collect rules reachable from trigger rules - for (const auto & [name, id] : rules_) { - const auto & parser = parsers_.at(id); - if (auto rule = std::get_if(&parser)) { - if (rule->trigger) { - // Mark trigger as reachable and visit it - reachable_rules.insert(name); - auto add_rules = collect_reachable_rules(*this, id); - reachable_rules.insert(add_rules.begin(), add_rules.end()); - } + // Trigger rules, sorted for a predictable order + std::vector trigger_rules; + for (const auto & [name, id] : rules_) { + if (auto rule = std::get_if(&parsers_.at(id))) { + if (rule->trigger) { + trigger_rules.push_back(rule); } } + } + std::sort(trigger_rules.begin(), trigger_rules.end(), [](const auto * a, const auto * b) { return a->name < b->name; }); + + // A lazy grammar scans for the starts when the trigger rules were built from a start and rest + bool scans = false; + for (const auto * rule : trigger_rules) { + scans = scans || rule->start != COMMON_PEG_INVALID_PARSER_ID; + } + if (scans) { + for (const auto * rule : trigger_rules) { + if (rule->start == COMMON_PEG_INVALID_PARSER_ID) { + throw std::runtime_error("trigger rule without a start in a scanning grammar: " + rule->name); + } + } + } + + if (lazy && scans) { + for (const auto * rule : trigger_rules) { + collect_reachable(*this, rule->start, reachable); + collect_reachable(*this, rule->rest, reachable); + } + } else if (lazy) { + for (const auto * rule : trigger_rules) { + collect_reachable(*this, rules_.at(rule->name), reachable); + } } else { - // Collect rules reachable from root - reachable_rules = collect_reachable_rules(*this, root_); + collect_reachable(*this, root_, reachable); } // Create GBNF rules for all reachable rules for (const auto & [name, rule_id] : rules_) { - if (reachable_rules.find(name) == reachable_rules.end()) { + if (reachable.rules.find(name) == reachable.rules.end()) { continue; } @@ -1807,20 +2228,27 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo } } - if (lazy) { - // Generate root rule from trigger rules only - std::vector trigger_names; - for (const auto & [name, rule_id] : rules_) { - const auto & parser = parsers_.at(rule_id); - if (auto rule = std::get_if(&parser)) { - if (rule->trigger) { - trigger_names.push_back(rule->name); + if (lazy && scans) { + // The root scans for a trigger start and then constrains its rest + common_trie starts; + std::vector rests; + for (const auto * rule : trigger_rules) { + auto value = (int32_t) rests.size(); + rests.push_back(builder.add_rule(rule->name + "-rest", to_gbnf(rule->rest))); + for (size_t end : insert_delimiter(starts, *this, rule->start)) { + auto owner = starts.mark(end, value); + if (owner != value) { + throw std::runtime_error("trigger rules " + trigger_rules[owner]->name + " and " + rule->name + " must not share a start"); } } } - - // Sort for predictable order - std::sort(trigger_names.begin(), trigger_names.end()); + builder.add_rule("root", gbnf_including_grammar(builder, "trigger", starts, reachable.tokens, rests, /* optional = */ true)); + } else if (lazy) { + // Generate root rule from trigger rules only + std::vector trigger_names; + for (const auto * rule : trigger_rules) { + trigger_names.push_back(rule->name); + } builder.add_rule("root", string_join(trigger_names, " | ")); } else if (root_ != COMMON_PEG_INVALID_PARSER_ID) { builder.add_rule("root", to_gbnf(root_)); diff --git a/common/peg-parser.h b/common/peg-parser.h index 46df99875..b7e97083b 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -3,6 +3,7 @@ #include "json-schema.h" #include "json.h" #include "llama.h" +#include "trie.h" #include #include @@ -188,6 +189,22 @@ struct common_peg_recursion_error : std::invalid_argument { using std::invalid_argument::invalid_argument; }; +struct common_peg_token { + std::string text; + llama_token_attr attr; +}; + +struct common_peg_tokens { + std::unordered_map tokens; + std::unordered_map ids; + + common_peg_tokens() = default; + explicit common_peg_tokens(const llama_vocab * vocab); + + // LLAMA_TOKEN_NULL if text is not a special token + llama_token token_id(const std::string & text) const; +}; + struct common_peg_parse_context { std::string input; // [h, e, l, l, o, _, w, o, r, l, d] std::vector tokens; // [id, -1, -1, -1, -1, id, -1, -1, -1, -1, -1] @@ -226,6 +243,16 @@ struct common_peg_literal_parser { std::string literal; }; +struct common_peg_token_parser { + llama_token token; + std::string piece; +}; + +// Matches one token that is not one of the tokens +struct common_peg_token_not_parser { + std::vector tokens; +}; + struct common_peg_sequence_parser { std::vector children; }; @@ -270,8 +297,18 @@ struct common_peg_string_parser { char delimiter; }; +// A delimiter an until parser stops at, and the parser that follows it when the delimiter is consumed +struct common_peg_until_branch { + common_peg_parser_id delimiter; + common_peg_parser_id rest = COMMON_PEG_INVALID_PARSER_ID; +}; + struct common_peg_until_parser { - std::vector delimiters; + std::vector branches; + bool include = false; // consume the first delimiter and parse the rest of its branch + bool optional = false; // with include, the input may end before any delimiter + common_peg_parser_id content = COMMON_PEG_INVALID_PARSER_ID; // with include, parsed up to the delimiter + common_trie matcher; // each pattern valued by its branch }; struct common_peg_schema_parser { @@ -288,6 +325,9 @@ struct common_peg_rule_parser { std::string name; common_peg_parser_id child; bool trigger; + // set when the rule has a start and rest, the lazy grammar scans for the start + common_peg_parser_id start = COMMON_PEG_INVALID_PARSER_ID; + common_peg_parser_id rest = COMMON_PEG_INVALID_PARSER_ID; }; struct common_peg_ref_parser { @@ -308,17 +348,14 @@ struct common_peg_gbnf_parser { std::string grammar; }; -struct common_peg_ac_parser { - common_peg_parser_id child; - std::vector delimiters; -}; - // Variant holding all parser types using common_peg_parser_variant = std::variant< common_peg_epsilon_parser, common_peg_start_parser, common_peg_end_parser, common_peg_literal_parser, + common_peg_token_parser, + common_peg_token_not_parser, common_peg_sequence_parser, common_peg_choice_parser, common_peg_repetition_parser, @@ -334,14 +371,14 @@ using common_peg_parser_variant = std::variant< common_peg_ref_parser, common_peg_atomic_parser, common_peg_tag_parser, - common_peg_gbnf_parser, - common_peg_ac_parser + common_peg_gbnf_parser >; class common_peg_arena { std::vector parsers_; std::unordered_map rules_; common_peg_parser_id root_ = COMMON_PEG_INVALID_PARSER_ID; + common_peg_tokens tokens_; public: const common_peg_parser_variant & get(common_peg_parser_id id) const { return parsers_.at(id); } @@ -356,6 +393,8 @@ class common_peg_arena { common_peg_parser_id root() const { return root_; } void set_root(common_peg_parser_id id) { root_ = id; } + const common_peg_tokens & tokens() const { return tokens_; } + common_peg_parse_result parse(common_peg_parse_context & ctx, size_t start = 0) const; common_peg_parse_result parse(common_peg_parser_id id, common_peg_parse_context & ctx, size_t start) const; @@ -384,6 +423,7 @@ class common_peg_parser_builder { public: common_peg_parser_builder(); + explicit common_peg_parser_builder(common_peg_tokens tokens); // Match nothing, always succeed. // S -> ε @@ -401,6 +441,16 @@ class common_peg_parser_builder { // S -> "hello" common_peg_parser literal(const std::string & literal) { return add(common_peg_literal_parser{literal}); } + // Matches a token. Falls back to a literal if the token is not registered with the builder. + // S -> <[token-id]> + common_peg_parser token(const std::string & piece); + + // Matches one of the tokens. With negate, matches one token that is not in the set. + // A negated set may only name tokens registered with the builder. + // S -> <[id1]> | <[id2]> | ... or !<[id1,id2,...]> + common_peg_parser token(const std::vector & pieces, bool negate = false); + common_peg_parser token(std::initializer_list pieces, bool negate = false) { return token(std::vector(pieces), negate); } + // Matches a sequence of parsers in order, all must succeed. // S -> A B C common_peg_parser sequence() { return add(common_peg_sequence_parser{}); } @@ -457,16 +507,29 @@ class common_peg_parser_builder { // Matches all characters until a delimiter is found (delimiter not consumed). // Invalid UTF-8 is consumed and recorded on the AST nodes. // S -> (!delim .)* - common_peg_parser until(const std::string & delimiter) { return add(common_peg_until_parser{{delimiter}}); } + common_peg_parser until(const std::string & delimiter) { return until_one_of({delimiter}); } // Matches all characters until one of the delimiters in the list is found (delimiter not consumed). // S -> (!delim .)* - common_peg_parser until_one_of(const std::vector & delimiters) { return add(common_peg_until_parser{delimiters}); } + common_peg_parser until_one_of(const std::vector & delimiters); + + // Matches all characters until the delimiter is found (delimiter not consumed). + // The delimiter may only be a sequence of literals, tokens, and choices of them. Tags and atomics may wrap them. + // S -> (!delim .)* + common_peg_parser until(const common_peg_parser & delimiter); // Matches everything // S -> .* common_peg_parser rest() { return until_one_of({}); } + // Matches content, then the first delimiter, then the rest of the branch of that delimiter. + // Branches must not share a delimiter. With optional, the input may end before any delimiter. + // S -> C (D1 R1 | D2 R2 | ...) + common_peg_parser through(const common_peg_parser & content, const std::vector & branches, bool optional = false); + common_peg_parser through(const common_peg_parser & content, std::initializer_list branches, bool optional = false) { return through(content, std::vector(branches), optional); } + common_peg_parser through(const common_peg_parser & content, const common_peg_parser & delimiter) { return through(content, { { delimiter } }); } + common_peg_parser through(const common_peg_parser & content, const std::string & delimiter) { return through(content, literal(delimiter)); } + // Matches between min and max repetitions of a parser (inclusive). // S -> A{m,n} // Use -1 for max to represent unbounded repetition (equivalent to {m,}) @@ -537,6 +600,10 @@ class common_peg_parser_builder { common_peg_parser trigger_rule(const std::string & name, const common_peg_parser & p) { return rule(name, p, true); } common_peg_parser trigger_rule(const std::string & name, const std::function & builder) { return rule(name, builder, true); } + // Creates a trigger rule that matches start followed by rest. A lazy grammar scans for the start of any such rule, then only allows its rest. + // The start has the same limits as an until() delimiter. + common_peg_parser trigger_rule(const std::string & name, const common_peg_parser & start, const common_peg_parser & rest); + // Creates an atomic parser. Atomic parsers do not create an AST node if // the child results in a partial parse, i.e. NEEDS_MORE_INPUT. This is // intended for situations where partial output is undesirable. @@ -550,13 +617,6 @@ class common_peg_parser_builder { // the child's grammar. Parsing delegates entirely to the child. common_peg_parser gbnf(const common_peg_parser & p, const std::string & grammar) { return add(common_peg_gbnf_parser{p, grammar}); } - // Wraps a child parser but emits a GBNF grammar built from the Aho-Corasick - // automaton of `delimiters`, matching everything up to and including the - // first delimiter. Parsing delegates entirely to the child, which is - // responsible for consuming the delimiter (e.g. until(D) + literal(D)). - common_peg_parser ac(const common_peg_parser & p, const std::vector & delimiters); - common_peg_parser ac(const common_peg_parser & p, const std::string & delimiter) { return ac(p, std::vector{delimiter}); } - void set_root(const common_peg_parser & p); common_peg_arena build(); diff --git a/common/reasoning-budget.cpp b/common/reasoning-budget.cpp index 81b7c3e56..ac2cd261d 100644 --- a/common/reasoning-budget.cpp +++ b/common/reasoning-budget.cpp @@ -31,14 +31,18 @@ struct token_matcher { static common_trie build_trie(const std::vector & seqs) { common_trie t; for (const auto & seq : seqs) { - t.insert(std::vector(seq.begin(), seq.end())); + std::vector symbols; + for (llama_token token : seq) { + symbols.push_back(common_trie::symbol::token(token)); + } + t.insert(symbols); } return t; } // returns the index into seqs of the longest sequence ending at this token, or -1 int32_t advance(llama_token token) { - state = ac.next(state, (uint32_t) token); + state = ac.next(state, common_trie::symbol::token(token)); const int32_t p = ac.match_pattern(state); if (p >= 0) { state = 0; diff --git a/common/trie.cpp b/common/trie.cpp index b5c9666ba..b009b7044 100644 --- a/common/trie.cpp +++ b/common/trie.cpp @@ -4,26 +4,46 @@ #include +common_trie::step common_trie::next(std::string_view sv, const std::vector & token_map, size_t pos) const { + if (!tokens.empty() && pos < token_map.size()) { + const int32_t id = token_map[pos]; + if (id >= 0 && tokens.find(id) != tokens.end()) { + size_t end = pos + 1; + while (end < token_map.size() && token_map[end] < 0) { + ++end; + } + return { utf8_parse_result::SUCCESS, symbol::token(id), end - pos }; + } + } + + auto result = common_parse_utf8_codepoint(sv, pos); + return { result.status, symbol::codepoint(result.codepoint), result.bytes_consumed }; +} + common_trie::match_result common_trie::check_at(std::string_view sv, size_t start_pos) const { + return check_at(sv, {}, start_pos); +} + +common_trie::match_result common_trie::check_at(std::string_view sv, const std::vector & token_map, size_t start_pos) const { size_t current = 0; // Start at root size_t pos = start_pos; // LOG_DBG("%s: checking at pos %zu, sv='%s'\n", __func__, start_pos, std::string(sv).c_str()); while (pos < sv.size()) { - auto result = common_parse_utf8_codepoint(sv, pos); - if (result.status != utf8_parse_result::SUCCESS) { + auto step = next(sv, token_map, pos); + if (step.status != utf8_parse_result::SUCCESS) { break; } - auto it = nodes[current].children.find(result.codepoint); + auto it = nodes[current].children.find(step.symbol); if (it == nodes[current].children.end()) { // Can't continue matching return match_result{match_result::NO_MATCH}; } current = it->second; - pos += result.bytes_consumed; + pos += step.bytes_consumed; // Check if we've matched a complete word if (nodes[current].pattern >= 0) { @@ -42,7 +62,7 @@ common_trie::match_result common_trie::check_at(std::string_view sv, size_t star } int32_t common_trie::insert(const std::string & word) { - std::vector symbols; + std::vector symbols; size_t pos = 0; while (pos < word.length()) { auto result = common_parse_utf8_codepoint(word, pos); @@ -50,28 +70,42 @@ int32_t common_trie::insert(const std::string & word) { break; } - symbols.push_back(result.codepoint); + symbols.push_back(symbol::codepoint(result.codepoint)); pos += result.bytes_consumed; } return insert(symbols); } -int32_t common_trie::insert(const std::vector & symbols) { +int32_t common_trie::insert(const std::vector & symbols) { size_t current = 0; - for (uint32_t ch : symbols) { - auto it = nodes[current].children.find(ch); - if (it == nodes[current].children.end()) { - size_t child = create_node(); - nodes[current].children[ch] = child; - current = child; - } else { - current = it->second; - } + for (symbol ch : symbols) { + current = extend(current, ch); } - if (nodes[current].pattern < 0) { - nodes[current].pattern = n_patterns++; + int32_t pattern = mark(current, n_patterns); + if (pattern == n_patterns) { + n_patterns++; } - return nodes[current].pattern; + return pattern; +} + +size_t common_trie::extend(size_t node, symbol ch) { + if (ch.is_token()) { + tokens.insert((int32_t) ch.value); + } + auto it = nodes[node].children.find(ch); + if (it != nodes[node].children.end()) { + return it->second; + } + size_t child = create_node(); + nodes[node].children[ch] = child; + return child; +} + +int32_t common_trie::mark(size_t node, int32_t value) { + if (nodes[node].pattern < 0) { + nodes[node].pattern = value; + } + return nodes[node].pattern; } common_aho_corasick::common_aho_corasick(common_trie trie) : t(std::move(trie)) { @@ -113,7 +147,7 @@ common_aho_corasick::common_aho_corasick(common_trie trie) : t(std::move(trie)) } } -size_t common_aho_corasick::next(size_t state, uint32_t ch) const { +size_t common_aho_corasick::next(size_t state, common_trie::symbol ch) const { const auto & nodes = t.nodes; while (state && nodes[state].children.find(ch) == nodes[state].children.end()) { state = fail[state]; diff --git a/common/trie.h b/common/trie.h index 0f7b16a36..8c31d7263 100644 --- a/common/trie.h +++ b/common/trie.h @@ -1,5 +1,7 @@ #pragma once +#include "unicode.h" + #include #include #include @@ -10,13 +12,42 @@ // Trie for matching multiple literals. // This is used in common_peg_until_parser and to build a GBNF exclusion grammar struct common_trie { + // A symbol is either a UTF-8 codepoint or a token + struct symbol { + enum kind_type : uint8_t { CODEPOINT, TOKEN }; + + kind_type kind = CODEPOINT; + uint32_t value = 0; + + static symbol codepoint(uint32_t cpt) { return { CODEPOINT, cpt }; } + static symbol token(int32_t id) { return { TOKEN, (uint32_t) id }; } + + bool is_token() const { return kind == TOKEN; } + + bool operator<(const symbol & other) const { + return kind != other.kind ? kind < other.kind : value < other.value; + } + bool operator==(const symbol & other) const { + return kind == other.kind && value == other.value; + } + }; + + struct step { + enum utf8_parse_result::status status; + common_trie::symbol symbol; + size_t bytes_consumed; + }; + struct node { - std::map children; // Use uint32_t to store Unicode codepoints + std::map children; int32_t pattern = -1; // index of the pattern ending at this node, -1 if none }; std::vector nodes; + // every token id used by a pattern, the input is split so these match as single symbols + std::set tokens; + common_trie() { create_node(); // root node } @@ -29,15 +60,25 @@ struct common_trie { enum match_result { NO_MATCH, PARTIAL_MATCH, COMPLETE_MATCH }; + // Read the symbol at pos. It is a token if a pattern uses the token at pos, else a UTF-8 codepoint. + step next(std::string_view sv, const std::vector & token_map, size_t pos) const; + // Check if a delimiter starts at the given position match_result check_at(std::string_view sv, size_t start_pos) const; + match_result check_at(std::string_view sv, const std::vector & token_map, size_t start_pos) const; // Insert a word as a sequence of Unicode codepoints, returns its pattern index int32_t insert(const std::string & word); - // Insert a raw symbol sequence, returns its pattern index (insertion order, - // duplicates keep the first index) - int32_t insert(const std::vector & symbols); + // Insert a symbol sequence, returns its pattern index (insertion order, duplicates keep the first index) + int32_t insert(const std::vector & symbols); + + // Child of node on a symbol, created if missing + size_t extend(size_t node, symbol ch); + + // Mark node as the end of a pattern with a value from the caller. Returns the value already there, or value if the node had none. + // A trie must take its values from mark or from insert, not both. + int32_t mark(size_t node, int32_t value); private: int32_t n_patterns = 0; @@ -51,11 +92,11 @@ struct common_trie { // Aho-Corasick automaton struct common_aho_corasick { - common_trie t; - std::vector fail; // failure links - std::vector order; // states in BFS order - std::vector match; // longest pattern ending at each state (directly or via a suffix link), -1 if none - std::set alphabet; // every character with a transition + common_trie t; + std::vector fail; // failure links + std::vector order; // states in BFS order + std::vector match; // longest pattern ending at each state (directly or via a suffix link), -1 if none + std::set alphabet; // every symbol with a transition common_aho_corasick(common_trie trie); @@ -69,5 +110,5 @@ struct common_aho_corasick { int32_t match_pattern(size_t s) const { return match[s]; } // follow failure links until a transition on `ch` exists. - size_t next(size_t state, uint32_t ch) const; + size_t next(size_t state, common_trie::symbol ch) const; }; diff --git a/docs/development/parsing.md b/docs/development/parsing.md index ab0ccad80..faa64afde 100644 --- a/docs/development/parsing.md +++ b/docs/development/parsing.md @@ -67,6 +67,8 @@ For a more complete example, see `test_example_native()` in - **`end()`** - Matches the end of input (anchor `$`) - **`literal(string)`** - Matches an exact literal string - **`any()`** - Matches any single character (`.`) +- **`token(piece)`** - Matches a special token, or the literal `piece` if it is not registered with the builder (see [Special Tokens](#special-tokens)) +- **`token(pieces, negate)`** - Matches one of the tokens, or with `negate` one token that is not one of them ### Combinators @@ -89,6 +91,8 @@ For a more complete example, see `test_example_native()` in - **`space()`** - Matches zero or more whitespace characters (space, tab, newline) - **`until(delimiter)`** - Matches characters until delimiter is found (delimiter not consumed) - **`until_one_of(delimiters)`** - Matches characters until any delimiter in the list is found +- **`until(parser)`** - Matches characters until a delimiter built from literals and tokens is found (delimiter not consumed, see [until(parser)](#untilparser)) +- **`through(content, branches, optional)`** - Matches content, then the first delimiter found and the rest of its branch (delimiter consumed). The grammar scans for the delimiters instead of following content - **`rest()`** - Matches everything remaining (`.*`) ### JSON Parsers @@ -108,6 +112,7 @@ For a more complete example, see `test_example_native()` in - **`ref(name)`** - Creates a lightweight reference to a named rule (for recursive grammars) - **`rule(name, p, trigger)`** - Creates a named rule and returns a reference - **`trigger_rule(name, p)`** - Creates a trigger rule (entry point for lazy grammar generation) +- **`trigger_rule(name, start, rest)`** - Creates a trigger rule matching `start` then `rest`, gating the lazy grammar on the start - **`schema(p, name, schema, raw)`** - Wraps parser with JSON schema metadata for grammar generation - **`schema(p, name, doc, node, raw)`** - Same, for a node of a `common_chat_schema_document` built earlier, e.g. one tool parameter @@ -116,6 +121,110 @@ For a more complete example, see `test_example_native()` in - **`atomic(p)`** - Prevents AST node creation for partial parses - **`tag(tag, p)`** - Creates AST nodes with semantic tags (multiple nodes can share tags) +## Special Tokens + +Matching a special token by its text alone is ambiguous: a model may emit +`<|end|>` as a single control token, or spell it out as plain text, such as +when quoting a chat template. The parser can tell them apart when the builder +knows the special tokens of the vocabulary, and the input carries the ids of +the tokens that produced it. + +A `common_peg_tokens` table constructed from a `llama_vocab` registers every +control and user-defined token. Pass it to the builder: + +```cpp +common_peg_tokens specials(vocab); + +auto parser = build_chat_peg_parser(specials, [&](common_chat_peg_builder & p) { + return p.content(p.until(p.token("<|end|>"))) + p.token("<|end|>"); +}); +``` + +Chat templates initialized with a model build this table from its vocabulary, +and format handlers receive it as `inputs.specials`. + +### token() + +`token(piece)` matches only where the input holds the id of that token, and +consumes the whole piece. The same characters without the id do not match. +If `piece` is not registered with the builder, it falls back to +`literal(piece)`. + +`token(pieces)` matches any of the tokens, and `token(pieces, true)` matches one +token that is not in the set. Every piece of a negated set must be registered, +otherwise it throws `std::invalid_argument`. + +In the grammar, a token emits `<[id]>` and a negated set emits `!<[...]>`: + +```cpp +p.token({ "", "" }) + p.token({ "", "" }, true) +``` + +``` +root ::= (<[1]> | <[2]>) !<[1-2]> +``` + +### until(parser) + +`until(delimiter)` with a parser instead of a string stops at a delimiter that +may contain tokens, or several literals joined in a sequence: + +```cpp +auto end = p.token("<|end|>"); +auto final = p.token("<|channel|>") + p.literal("final"); + +p.reasoning(p.until(end | final)) +``` + +The delimiter may only be built from: + +- `literal()` and `token()` +- `sequence()` and `choice()` of the above +- `tag()` and `atomic()` wrapping any of the above + +Any other parser throws `std::invalid_argument`. A choice inside a sequence +expands to every combination, so `"a" + ("b" | "d")` stops at both `ab` and +`ad`. + +The grammar scans for the delimiters the same way as `until_one_of()`: + +```cpp +p.until((p.literal("a") + p.literal("b")) | p.literal("cd")) +``` + +``` +root ::= until-5 +until-5 ::= | [a] until-5-01 | [c] until-5-03 | [^ac] until-5 +until-5-01 ::= | [a] until-5-01 | [c] until-5-03 | [^abc] until-5 +until-5-03 ::= | [a] until-5-01 | [c] until-5-03 | [^acd] until-5 +``` + +A token never matches a character rule such as `.` or `[^a]`, so the scan lets +the other tokens named elsewhere in the grammar through by name. Here `` is +token 2: + +```cpp +p.until(p.token("") + p.literal("x")) + p.token("") + p.literal("x") + p.token("") +``` + +``` +root ::= until-3 <[1]> "x" <[2]> +until-3 ::= | <[1]> until-3-01 | (. | <[2]>) until-3 +until-3-01 ::= | <[1]> until-3-01 | ([^x] | <[2]>) until-3 +``` + +When the delimiters are made of tokens alone, the scan consumes whole tokens +with a negated token set instead: + +```cpp +p.until(p.token("")) + p.token("") +``` + +``` +root ::= until-1 <[1]> +until-1 ::= | !<[1]> until-1 +``` + ## GBNF Grammar Generation The PEG parser also acts as a convenient DSL for generating GBNF grammars, with @@ -143,6 +252,28 @@ are emitted in the grammar. All trigger rules are added as alternations in the root rule. It is still necessary to define trigger patterns, as the parser has no interaction with the grammar sampling. +With `trigger_rule(name, start, rest)`, the grammar instead scans for a start and +then constrains the rest. It runs from the first token, so `grammar_lazy` is false +and no trigger patterns are needed. For example: + +```cpp +auto ab = p.literal("ab"); +auto cd = p.literal("cd"); +return p.until(ab | cd) + (p.trigger_rule("one", ab, p.literal("x")) | p.trigger_rule("two", cd, p.literal("y"))); +``` + +``` +root ::= trigger +trigger ::= | [a] trigger-01 | [c] trigger-03 | [^ac] trigger +trigger-01 ::= | [b] one-rest | [a] trigger-01 | [c] trigger-03 | [^abc] trigger +trigger-03 ::= | [d] two-rest | [a] trigger-01 | [c] trigger-03 | [^acd] trigger +one-rest ::= "x" +two-rest ::= "y" +``` + +A start may only be a sequence of literals, tokens, and choices of them, possibly +wrapped in tags and atomics. + ### JSON Schema The `schema(p, name, schema, raw)` parser will use the `json-schema-to-grammar` diff --git a/grammars/README.md b/grammars/README.md index f005fc252..b35946a4a 100644 --- a/grammars/README.md +++ b/grammars/README.md @@ -73,11 +73,13 @@ Tokens allow grammars to match specific tokenizer tokens rather than character s Tokens can be specified in two ways: -1. **Token ID**: Use angle brackets with the token ID in square brackets: `<[token-id]>`. For example, `<[1000]>` matches the token with ID 1000. +1. **Token ID**: Use angle brackets with the token ID in square brackets: `<[token-id]>`. For example, `<[1000]>` matches the token with ID 1000. Multiple IDs and inclusive ranges can be combined: `<[1000,1005-1010]>` matches token 1000 or any token from 1005 to 1010. 2. **Token string**: Use angle brackets with the token text directly: ``. For example, `` will match the token whose text is exactly ``. This only works if the string tokenizes to exactly one token in the vocabulary, otherwise the grammar will fail to parse. -You can negate token matches using the `!` prefix: `!<[1000]>` or `!` matches any token *except* the specified one. +You can negate token matches using the `!` prefix: `!<[1000]>` or `!` matches any token *except* the specified one, and `!<[1000,1005-1010]>` matches any token outside the set. + +Tokens are matched whole, so a token rule can only match at the start of a token, never after a character rule has consumed part of its text. Any token named by a token rule is also excluded from character rules: once `` appears as a token in a grammar, `.*` in that grammar will not match the `` token. ``` # Match a thinking block: ... diff --git a/src/llama-grammar.cpp b/src/llama-grammar.cpp index 9a0de1430..21a018c8c 100644 --- a/src/llama-grammar.cpp +++ b/src/llama-grammar.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -33,9 +34,9 @@ static std::pair decode_utf8(const char * src) { return std::make_pair(value, pos); } -static std::pair, llama_partial_utf8> decode_utf8( - const std::string & src, - llama_partial_utf8 partial_start) { +std::pair, llama_partial_utf8> llama_grammar_decode_utf8( + const std::string & src, + llama_partial_utf8 partial_start) { static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 2, 2, 3, 4 }; const char * pos = src.c_str(); std::vector code_points; @@ -185,23 +186,45 @@ static std::pair parse_char(const char * src) { throw std::runtime_error("unexpected end of input"); } -static std::pair parse_token(const llama_vocab * vocab, const char * src) { +static std::pair parse_token_id(const char * src) { + const char * int_end = parse_int(src); + unsigned long id = std::stoul(std::string(src, int_end - src)); + if (id > std::numeric_limits::max()) { + throw std::runtime_error(std::string("parsed token id is too big at ") + src); + } + return std::make_pair(static_cast(id), int_end); +} + +static const char * parse_token(const llama_vocab * vocab, const char * src, llama_gretype type, llama_grammar_rule & rule) { const char * pos = src; if (*pos != '<') { throw std::runtime_error(std::string("expecting '<' at ") + pos); } pos++; - // Parse <[id]> + // Parse <[id,id-id,...]> if (*pos == '[') { pos++; - const char * int_end = parse_int(pos); - unsigned long id = std::stoul(std::string(pos, int_end - pos)); - if (id > std::numeric_limits::max()) { - throw std::runtime_error(std::string("parsed token id is too big at ") + pos); + const size_t start = rule.size(); + while (true) { + auto id_pair = parse_token_id(pos); + uint32_t token_id = id_pair.first; + pos = id_pair.second; + rule.push_back({rule.size() == start ? type : LLAMA_GRETYPE_TOKEN_ALT, token_id}); + if (*pos == '-') { + pos++; + auto end_pair = parse_token_id(pos); + if (end_pair.first < token_id) { + throw std::runtime_error(std::string("invalid token range at ") + pos); + } + pos = end_pair.second; + rule.push_back({LLAMA_GRETYPE_TOKEN_RNG_UPPER, end_pair.first}); + } + if (*pos != ',') { + break; + } + pos++; } - uint32_t token_id = static_cast(id); - pos = int_end; if (*pos != ']') { throw std::runtime_error(std::string("expecting ']' at ") + pos); } @@ -210,7 +233,7 @@ static std::pair parse_token(const llama_vocab * vocab, throw std::runtime_error(std::string("expecting '>' at ") + pos); } pos++; - return std::make_pair(token_id, pos); + return pos; } if (vocab == nullptr) { @@ -232,7 +255,8 @@ static std::pair parse_token(const llama_vocab * vocab, // must tokenize to exactly 1 token throw std::runtime_error("invalid token '" + std::string(src, pos - src) + "'"); } - return std::make_pair(tokens[0], pos); + rule.push_back({type, static_cast(tokens[0])}); + return pos; } static void print_grammar_char(FILE * file, uint32_t c) { @@ -244,6 +268,16 @@ static void print_grammar_char(FILE * file, uint32_t c) { } } +static bool is_token_element(llama_grammar_element elem) { + switch (elem.type) { + case LLAMA_GRETYPE_TOKEN: return true; + case LLAMA_GRETYPE_TOKEN_NOT: return true; + case LLAMA_GRETYPE_TOKEN_ALT: return true; + case LLAMA_GRETYPE_TOKEN_RNG_UPPER: return true; + default: return false; + } +} + static bool is_char_element(llama_grammar_element elem) { switch (elem.type) { case LLAMA_GRETYPE_CHAR: return true; @@ -258,16 +292,18 @@ static bool is_char_element(llama_grammar_element elem) { static void print_rule_binary(FILE * file, const llama_grammar_rule & rule) { for (auto elem : rule) { switch (elem.type) { - case LLAMA_GRETYPE_END: fprintf(file, "END"); break; - case LLAMA_GRETYPE_ALT: fprintf(file, "ALT"); break; - case LLAMA_GRETYPE_RULE_REF: fprintf(file, "RULE_REF"); break; - case LLAMA_GRETYPE_CHAR: fprintf(file, "CHAR"); break; - case LLAMA_GRETYPE_CHAR_NOT: fprintf(file, "CHAR_NOT"); break; - case LLAMA_GRETYPE_CHAR_RNG_UPPER: fprintf(file, "CHAR_RNG_UPPER"); break; - case LLAMA_GRETYPE_CHAR_ALT: fprintf(file, "CHAR_ALT"); break; - case LLAMA_GRETYPE_CHAR_ANY: fprintf(file, "CHAR_ANY"); break; - case LLAMA_GRETYPE_TOKEN: fprintf(file, "TOKEN"); break; - case LLAMA_GRETYPE_TOKEN_NOT: fprintf(file, "TOKEN_NOT"); break; + case LLAMA_GRETYPE_END: fprintf(file, "END"); break; + case LLAMA_GRETYPE_ALT: fprintf(file, "ALT"); break; + case LLAMA_GRETYPE_RULE_REF: fprintf(file, "RULE_REF"); break; + case LLAMA_GRETYPE_CHAR: fprintf(file, "CHAR"); break; + case LLAMA_GRETYPE_CHAR_NOT: fprintf(file, "CHAR_NOT"); break; + case LLAMA_GRETYPE_CHAR_RNG_UPPER: fprintf(file, "CHAR_RNG_UPPER"); break; + case LLAMA_GRETYPE_CHAR_ALT: fprintf(file, "CHAR_ALT"); break; + case LLAMA_GRETYPE_CHAR_ANY: fprintf(file, "CHAR_ANY"); break; + case LLAMA_GRETYPE_TOKEN: fprintf(file, "TOKEN"); break; + case LLAMA_GRETYPE_TOKEN_NOT: fprintf(file, "TOKEN_NOT"); break; + case LLAMA_GRETYPE_TOKEN_RNG_UPPER: fprintf(file, "TOKEN_RNG_UPPER"); break; + case LLAMA_GRETYPE_TOKEN_ALT: fprintf(file, "TOKEN_ALT"); break; } switch (elem.type) { case LLAMA_GRETYPE_END: @@ -295,6 +331,10 @@ static void print_rule_binary(FILE * file, const llama_grammar_rule & rule) { fprintf(file, "%u", elem.value); fprintf(file, "]> "); break; + case LLAMA_GRETYPE_TOKEN_RNG_UPPER: + case LLAMA_GRETYPE_TOKEN_ALT: + fprintf(file, "(%u) ", elem.value); + break; } } fprintf(file, "\n"); @@ -354,14 +394,32 @@ static void print_rule( case LLAMA_GRETYPE_TOKEN: fprintf(file, "<["); fprintf(file, "%u", elem.value); - fprintf(file, "]> "); break; case LLAMA_GRETYPE_TOKEN_NOT: fprintf(file, "!"); fprintf(file, "<["); fprintf(file, "%u", elem.value); - fprintf(file, "]> "); break; + case LLAMA_GRETYPE_TOKEN_RNG_UPPER: + if (i == 0 || !is_token_element(rule[i - 1])) { + throw std::runtime_error( + "LLAMA_GRETYPE_TOKEN_RNG_UPPER without preceding token: " + + std::to_string(rule_id) + "," + std::to_string(i)); + } + fprintf(file, "-%u", elem.value); + break; + case LLAMA_GRETYPE_TOKEN_ALT: + if (i == 0 || !is_token_element(rule[i - 1])) { + throw std::runtime_error( + "LLAMA_GRETYPE_TOKEN_ALT without preceding token: " + + std::to_string(rule_id) + "," + std::to_string(i)); + } + fprintf(file, ",%u", elem.value); + break; + } + if (is_token_element(elem) && + rule[i + 1].type != LLAMA_GRETYPE_TOKEN_ALT && rule[i + 1].type != LLAMA_GRETYPE_TOKEN_RNG_UPPER) { + fprintf(file, "]> "); } if (is_char_element(elem)) { switch (rule[i + 1].type) { @@ -587,11 +645,9 @@ const char * llama_grammar_parser::parse_sequence( type = LLAMA_GRETYPE_TOKEN_NOT; pos++; } - auto token_pair = parse_token(vocab, pos); - const char * token_end = token_pair.second; last_sym_start = rule.size(); n_prev_rules = 1; - rule.push_back({type, token_pair.first}); + const char * token_end = parse_token(vocab, pos, type, rule); pos = parse_space(token_end, is_nested); } else if (is_word_char(*pos)) { // rule reference const char * name_end = parse_name(pos); @@ -854,17 +910,58 @@ static bool llama_grammar_match_partial_char( // returns true iff token matches the rule at pos (regular or inverse) // asserts that pos is pointing to a token element -static bool llama_grammar_match_token( +static std::pair llama_grammar_match_token( const llama_grammar_element * pos, const llama_token token) { - GGML_ASSERT(pos->type == LLAMA_GRETYPE_TOKEN || pos->type == LLAMA_GRETYPE_TOKEN_NOT); - if (pos->type == LLAMA_GRETYPE_TOKEN) { - return pos->value == static_cast(token); + bool is_positive_token = pos->type == LLAMA_GRETYPE_TOKEN; + + GGML_ASSERT(is_positive_token || pos->type == LLAMA_GRETYPE_TOKEN_NOT); + + const uint32_t id = static_cast(token); + + bool found = false; + do { + if (pos[1].type == LLAMA_GRETYPE_TOKEN_RNG_UPPER) { + // inclusive range, e.g. <[1-5]> + found = found || (pos->value <= id && id <= pos[1].value); + pos += 2; + } else { + found = found || pos->value == id; + pos += 1; + } + } while (pos->type == LLAMA_GRETYPE_TOKEN_ALT); + + return std::make_pair(found == is_positive_token, pos); +} + +// every token id a token rule names, merged into sorted inclusive ranges +static std::vector> llama_grammar_opaque_tokens(const llama_grammar_rules & rules) { + std::vector> ranges; + for (const auto & rule : rules) { + for (size_t i = 0; i < rule.size(); i++) { + const auto type = rule[i].type; + if (type == LLAMA_GRETYPE_TOKEN || type == LLAMA_GRETYPE_TOKEN_NOT || type == LLAMA_GRETYPE_TOKEN_ALT) { + const bool is_range = i + 1 < rule.size() && rule[i + 1].type == LLAMA_GRETYPE_TOKEN_RNG_UPPER; + ranges.emplace_back(rule[i].value, is_range ? rule[i + 1].value : rule[i].value); + } + } } - if (pos->type == LLAMA_GRETYPE_TOKEN_NOT) { - return pos->value != static_cast(token); + std::sort(ranges.begin(), ranges.end()); + std::vector> merged; + for (const auto & r : ranges) { + if (!merged.empty() && r.first <= (uint64_t) merged.back().second + 1) { + merged.back().second = std::max(merged.back().second, r.second); + } else { + merged.push_back(r); + } } - return false; + return merged; +} + +bool llama_grammar_is_opaque(const llama_grammar & grammar, llama_token token) { + const auto id = static_cast(token); + auto it = std::upper_bound(grammar.opaque_tokens.begin(), grammar.opaque_tokens.end(), std::make_pair(id, UINT32_MAX)); + return it != grammar.opaque_tokens.begin() && std::prev(it)->second >= id; } // transforms a grammar pushdown stack into N possible stacks, all ending @@ -946,8 +1043,7 @@ static void llama_grammar_advance_stack( break; default: // end of alternate (LLAMA_GRETYPE_END, LLAMA_GRETYPE_ALT) or middle of char range - // (LLAMA_GRETYPE_CHAR_ALT, LLAMA_GRETYPE_CHAR_RNG_UPPER); stack should never be left on - // those + // (LLAMA_GRETYPE_CHAR_ALT, LLAMA_GRETYPE_CHAR_RNG_UPPER) or token set (LLAMA_GRETYPE_TOKEN_ALT, LLAMA_GRETYPE_TOKEN_RNG_UPPER); stack should never be left on those GGML_ABORT("fatal error"); } } @@ -1238,13 +1334,13 @@ static bool llama_grammar_reject_step( if (stack_pos->type == LLAMA_GRETYPE_TOKEN || stack_pos->type == LLAMA_GRETYPE_TOKEN_NOT) { for (const uint32_t i : cur) { const auto & tok = candidates[i]; - if (tok.code_points[depth] == 0) { - // reached the end of a token consumed by char rules, reject iff it ended - // in a partial response - if (tok.partial_utf8.n_remain != 0) { + if (depth > 0) { + // char rules already consumed part of this token, so it cannot match by id. + // accept it only if they consumed all of it and did not end in a partial sequence. + if (tok.code_points[depth] != 0 || tok.partial_utf8.n_remain != 0) { rejects.push_back(i); } - } else if (!llama_grammar_match_token(stack_pos, tok.id)) { + } else if (tok.partial_utf8.n_remain < 0 || !llama_grammar_match_token(stack_pos, tok.id).first) { rejects.push_back(i); } } @@ -1255,7 +1351,10 @@ static bool llama_grammar_reject_step( for (const uint32_t i : cur) { const auto & tok = candidates[i]; - if (tok.code_points[depth] == 0) { + if (tok.opaque) { + // a token named by a token rule only matches token rules + rejects.push_back(i); + } else if (tok.code_points[depth] == 0) { // reached end of full codepoints in token, reject iff it ended in a partial sequence // that cannot satisfy this position in grammar if (tok.partial_utf8.n_remain != 0 && @@ -1425,6 +1524,8 @@ struct llama_grammar * llama_grammar_init_impl( } } while (true); + auto opaque_tokens = llama_grammar_opaque_tokens(vec_rules); + // Important: vec_rules has to be moved here, not copied, because stacks contains // pointers to elements of vec_rules. If vec_rules were copied into llama_grammar // then the pointers would be invalidated when the local vec_rules goes out of scope. @@ -1439,6 +1540,7 @@ struct llama_grammar * llama_grammar_init_impl( /* .trigger_buffer_positions = */ {}, /* .trigger_tokens = */ {}, /* .trigger_patterns = */ {}, + /* .opaque_tokens = */ std::move(opaque_tokens), }; } @@ -1540,6 +1642,8 @@ struct llama_grammar * llama_grammar_init_impl( trigger.regex = std::regex(trigger.pattern); } + auto opaque_tokens = llama_grammar_opaque_tokens(vec_rules); + // Important: vec_rules has to be moved here, not copied, because stacks contains // pointers to elements of vec_rules. If vec_rules were copied into llama_grammar // then the pointers would be invalidated when the local vec_rules goes out of scope. @@ -1554,6 +1658,7 @@ struct llama_grammar * llama_grammar_init_impl( /* .trigger_buffer_positions = */ {}, std::move(vec_trigger_tokens), std::move(vec_trigger_patterns), + std::move(opaque_tokens), }; } @@ -1601,8 +1706,9 @@ void llama_grammar_apply_impl(const struct llama_grammar & grammar, llama_token_ } else if (piece.empty() || piece[0] == 0) { cur_p->data[i].logit = -INFINITY; } else { - candidates_decoded.push_back(decode_utf8(piece, grammar.partial_utf8)); - candidates_grammar.push_back({ i, candidates_decoded.back().first.data(), candidates_decoded.back().second, id }); + candidates_decoded.push_back(llama_grammar_decode_utf8(piece, grammar.partial_utf8)); + candidates_grammar.push_back({ i, candidates_decoded.back().first.data(), candidates_decoded.back().second, id, + llama_grammar_is_opaque(grammar, id) }); } } @@ -1673,7 +1779,7 @@ void llama_grammar_accept_impl(struct llama_grammar & grammar, llama_token token void llama_grammar_accept_str(struct llama_grammar & grammar, const std::string & piece) { // Note terminating 0 in decoded string - const auto decoded = decode_utf8(piece, grammar.partial_utf8); + const auto decoded = llama_grammar_decode_utf8(piece, grammar.partial_utf8); const auto & code_points = decoded.first; for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) { @@ -1688,9 +1794,16 @@ void llama_grammar_accept_str(struct llama_grammar & grammar, const std::string void llama_grammar_accept_token(struct llama_grammar & grammar, llama_token token, const std::string & piece) { // Note terminating 0 in decoded string - const auto decoded = decode_utf8(piece, grammar.partial_utf8); + const auto decoded = llama_grammar_decode_utf8(piece, grammar.partial_utf8); const auto & code_points = decoded.first; + // the mask rejects a piece that does not continue a partial character, so accepting one has to fail too + if (decoded.second.n_remain < 0) { + throw std::runtime_error("Invalid UTF-8 continuation in piece: " + piece + " (" + std::to_string(token) + ")"); + } + + const bool opaque = llama_grammar_is_opaque(grammar, token); + llama_grammar_stacks stacks_new; stacks_new.reserve(grammar.stacks.size()); @@ -1702,14 +1815,15 @@ void llama_grammar_accept_token(struct llama_grammar & grammar, llama_token toke const llama_grammar_element * pos = stack.back(); if (pos->type == LLAMA_GRETYPE_TOKEN || pos->type == LLAMA_GRETYPE_TOKEN_NOT) { - if (llama_grammar_match_token(pos, token)) { + auto match = llama_grammar_match_token(pos, token); + if (match.first) { llama_grammar_stack new_stack(stack.begin(), stack.end() - 1); - if (!llama_grammar_is_end_of_sequence(pos + 1)) { - new_stack.push_back(pos + 1); + if (!llama_grammar_is_end_of_sequence(match.second)) { + new_stack.push_back(match.second); } llama_grammar_advance_stack(*grammar.rules, new_stack, stacks_new); } - } else { + } else if (!opaque) { llama_grammar_stacks current_stacks = {stack}; for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) { diff --git a/src/llama-grammar.h b/src/llama-grammar.h index 5829d8edc..fedc70969 100644 --- a/src/llama-grammar.h +++ b/src/llama-grammar.h @@ -38,11 +38,17 @@ enum llama_gretype { // any character (.) LLAMA_GRETYPE_CHAR_ANY = 7, - // terminal element: token (<[token-id]>) + // terminal element: token (<[token-id]>, <[token-id,token-id-token-id]>) LLAMA_GRETYPE_TOKEN = 8, - // inverse token (!<[token-id]>) + // inverse token (!<[token-id]>, !<[token-id,token-id-token-id]>) LLAMA_GRETYPE_TOKEN_NOT = 9, + + // modifies a preceding LLAMA_GRETYPE_TOKEN, LLAMA_GRETYPE_TOKEN_NOT, or LLAMA_GRETYPE_TOKEN_ALT to be an inclusive range (<[1-5]>) + LLAMA_GRETYPE_TOKEN_RNG_UPPER = 10, + + // modifies a preceding LLAMA_GRETYPE_TOKEN, LLAMA_GRETYPE_TOKEN_NOT, or LLAMA_GRETYPE_TOKEN_RNG_UPPER to add an alternate token to match (<[1,5]>, <[1-5,9]>) + LLAMA_GRETYPE_TOKEN_ALT = 11, }; typedef struct llama_grammar_element { @@ -60,6 +66,7 @@ struct llama_grammar_candidate { const uint32_t * code_points; llama_partial_utf8 partial_utf8; llama_token id; + bool opaque; }; using llama_grammar_rule = std::vector< llama_grammar_element>; @@ -84,6 +91,10 @@ std::vector llama_grammar_reject_candidates_for_stack( const llama_grammar_stack & stack, const llama_grammar_candidates & candidates); +std::pair, llama_partial_utf8> llama_grammar_decode_utf8( + const std::string & src, + llama_partial_utf8 partial_start); + struct llama_grammar_parser { const llama_vocab * vocab; std::map symbol_ids; @@ -151,6 +162,8 @@ struct llama_grammar { trigger_patterns; // Regular expressions that trigger a lazy grammar. Must be a full match of the entire generated // string, and the grammar will be given the string from the first match group onwards. + // token ids named by token rules, as sorted inclusive ranges. char rules never match these tokens. + std::vector> opaque_tokens; }; // @@ -195,3 +208,6 @@ void llama_grammar_accept_token( struct llama_grammar & grammar, llama_token token, const std::string & piece); + +// true if a token rule names the token, char rules do not match it +bool llama_grammar_is_opaque(const llama_grammar & grammar, llama_token token); diff --git a/tests/peg-parser/test-basic.cpp b/tests/peg-parser/test-basic.cpp index b6af61491..df124c759 100644 --- a/tests/peg-parser/test-basic.cpp +++ b/tests/peg-parser/test-basic.cpp @@ -367,6 +367,49 @@ void test_basic(testing & t) { }); + t.test("through", [](testing & t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { + return p.through(p.tag("body", p.until_one_of({ "", "" })), { { p.literal(""), p.literal("x") }, { p.literal("") } }, true); + }); + + t.test("consumes the delimiter and its rest", [&](testing & t) { + auto ctx = common_peg_parse_context("hellox"); + auto result = parser.parse(ctx); + t.assert_equal("success", true, result.success()); + t.assert_equal("end", (size_t) 10, result.end); + t.assert_equal("body", std::string("hello"), std::string(ctx.ast.get(result.nodes[0]).text)); + }); + + t.test("picks the branch of the delimiter found", [&](testing & t) { + auto ctx = common_peg_parse_context("hellox"); + auto result = parser.parse(ctx); + t.assert_equal("success", true, result.success()); + t.assert_equal("end", (size_t) 9, result.end); + }); + + t.test("waits for the rest of a partial delimiter", [&](testing & t) { + auto ctx = common_peg_parse_context("hello"); + }); + auto ctx = common_peg_parse_context("hello"); + auto result = parser.parse(ctx); + t.assert_equal("fail", true, result.fail()); + }); + }); + t.test("recursive rules", [](testing &t) { // Test simple number t.test("simple_number", [](testing &t) { diff --git a/tests/peg-parser/test-gbnf-generation.cpp b/tests/peg-parser/test-gbnf-generation.cpp index 60066a817..38f3ec9ab 100644 --- a/tests/peg-parser/test-gbnf-generation.cpp +++ b/tests/peg-parser/test-gbnf-generation.cpp @@ -129,14 +129,14 @@ void test_gbnf_generation(testing &t) { }); assert_gbnf_equal(t, R"""( - root ::= until-0 + root ::= until-1 space ::= | " " | "\n"{1,2} [ \t]{0,20} - until-0 ::= | [<] until-0-01 | [^<] until-0 - until-0-01 ::= | [<] until-0-01 | [/] until-0-02 | [^/<] until-0 - until-0-02 ::= | [<] until-0-01 | [t] until-0-03 | [^] until-0 + until-1 ::= | [<] until-1-01 | [^<] until-1 + until-1-01 ::= | [<] until-1-01 | [/] until-1-02 | [^/<] until-1 + until-1-02 ::= | [<] until-1-01 | [t] until-1-03 | [^] until-1 )""", gbnf); }); @@ -150,22 +150,22 @@ void test_gbnf_generation(testing &t) { }); assert_gbnf_equal(t, R"""( - root ::= until-0 + root ::= until-1 space ::= | " " | "\n"{1,2} [ \t]{0,20} - until-0 ::= | [\n] until-0-01 | [^\n] until-0 - until-0-01 ::= | [\n] until-0-01 | [<] until-0-02 | [^\n<] until-0 - until-0-02 ::= | [\n] until-0-01 | [/] until-0-03 | [^\n/] until-0 - until-0-03 ::= | [\n] until-0-01 | [p] until-0-04 | [^\np] until-0 - until-0-04 ::= | [\n] until-0-01 | [a] until-0-05 | [^\na] until-0 - until-0-05 ::= | [\n] until-0-01 | [r] until-0-06 | [^\nr] until-0 - until-0-06 ::= | [\n] until-0-01 | [a] until-0-07 | [^\na] until-0 - until-0-07 ::= | [\n] until-0-01 | [m] until-0-08 | [^\nm] until-0 - until-0-08 ::= | [\n] until-0-01 | [e] until-0-09 | [^\ne] until-0 - until-0-09 ::= | [\n] until-0-01 | [t] until-0-10 | [^\nt] until-0 - until-0-10 ::= | [\n] until-0-01 | [e] until-0-11 | [^\ne] until-0 - until-0-11 ::= | [\n] until-0-01 | [r] until-0-12 | [^\nr] until-0 - until-0-12 ::= | [\n] until-0-01 | [>] until-0-13 | [^\n>] until-0 - until-0-13 ::= | [^\n] until-0 + until-1 ::= | [\n] until-1-01 | [^\n] until-1 + until-1-01 ::= | [\n] until-1-01 | [<] until-1-02 | [^\n<] until-1 + until-1-02 ::= | [\n] until-1-01 | [/] until-1-03 | [^\n/] until-1 + until-1-03 ::= | [\n] until-1-01 | [p] until-1-04 | [^\np] until-1 + until-1-04 ::= | [\n] until-1-01 | [a] until-1-05 | [^\na] until-1 + until-1-05 ::= | [\n] until-1-01 | [r] until-1-06 | [^\nr] until-1 + until-1-06 ::= | [\n] until-1-01 | [a] until-1-07 | [^\na] until-1 + until-1-07 ::= | [\n] until-1-01 | [m] until-1-08 | [^\nm] until-1 + until-1-08 ::= | [\n] until-1-01 | [e] until-1-09 | [^\ne] until-1 + until-1-09 ::= | [\n] until-1-01 | [t] until-1-10 | [^\nt] until-1 + until-1-10 ::= | [\n] until-1-01 | [e] until-1-11 | [^\ne] until-1 + until-1-11 ::= | [\n] until-1-01 | [r] until-1-12 | [^\nr] until-1 + until-1-12 ::= | [\n] until-1-01 | [>] until-1-13 | [^\n>] until-1 + until-1-13 ::= | [^\n] until-1 )""", gbnf); }); @@ -181,15 +181,15 @@ void test_gbnf_generation(testing &t) { }); assert_gbnf_equal(t, R"""( - root ::= until-0 + root ::= until-1 space ::= | " " | "\n"{1,2} [ \t]{0,20} - until-0 ::= | [<] until-0-01 | [^<] until-0 - until-0-01 ::= | [<] until-0-01 | [\uFF5C] until-0-02 | [^<\uFF5C] until-0 - until-0-02 ::= | [<] until-0-01 | [D] until-0-03 | [^") + p.literal(""), ""); + return p.through(p.until(""), ""); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { @@ -222,20 +222,20 @@ void test_gbnf_generation(testing &t) { }); assert_gbnf_equal(t, R"""( - ac-3 ::= [<] ac-3-01 | [^<] ac-3 - ac-3-01 ::= [<] ac-3-01 | [/] ac-3-02 | [^/<] ac-3 - ac-3-02 ::= [<] ac-3-01 | [t] ac-3-03 | [^] | [<] ac-3-01 | [^<>] ac-3 - root ::= ac-3 + root ::= until-3 space ::= | " " | "\n"{1,2} [ \t]{0,20} + until-3 ::= [<] until-3-01 | [^<] until-3 + until-3-01 ::= [<] until-3-01 | [/] until-3-02 | [^/<] until-3 + until-3-02 ::= [<] until-3-01 | [t] until-3-03 | [^] | [<] until-3-01 | [^<>] until-3 )""", gbnf); }); - t.test("ac grammar terminates at first delimiter", [](testing &t) { + t.test("through grammar terminates at first delimiter", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { - return p.ac(p.until("\n\n") + p.literal("\n\n"), "\n\n"); + return p.through(p.until("\n\n"), "\n\n"); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { @@ -243,28 +243,28 @@ void test_gbnf_generation(testing &t) { }); assert_gbnf_equal(t, R"""( - ac-3 ::= [\n] ac-3-01 | [^\n] ac-3 - ac-3-01 ::= [\n] ac-3-01 | [<] ac-3-02 | [^\n<] ac-3 - ac-3-02 ::= [\n] ac-3-01 | [/] ac-3-03 | [^\n/] ac-3 - ac-3-03 ::= [\n] ac-3-01 | [p] ac-3-04 | [^\np] ac-3 - ac-3-04 ::= [\n] ac-3-01 | [a] ac-3-05 | [^\na] ac-3 - ac-3-05 ::= [\n] ac-3-01 | [r] ac-3-06 | [^\nr] ac-3 - ac-3-06 ::= [\n] ac-3-01 | [a] ac-3-07 | [^\na] ac-3 - ac-3-07 ::= [\n] ac-3-01 | [m] ac-3-08 | [^\nm] ac-3 - ac-3-08 ::= [\n] ac-3-01 | [e] ac-3-09 | [^\ne] ac-3 - ac-3-09 ::= [\n] ac-3-01 | [t] ac-3-10 | [^\nt] ac-3 - ac-3-10 ::= [\n] ac-3-01 | [e] ac-3-11 | [^\ne] ac-3 - ac-3-11 ::= [\n] ac-3-01 | [r] ac-3-12 | [^\nr] ac-3 - ac-3-12 ::= [\n] ac-3-01 | [>] ac-3-13 | [^\n>] ac-3 - ac-3-13 ::= [\n] | [^\n] ac-3 - root ::= ac-3 + root ::= until-3 space ::= | " " | "\n"{1,2} [ \t]{0,20} + until-3 ::= [\n] until-3-01 | [^\n] until-3 + until-3-01 ::= [\n] until-3-01 | [<] until-3-02 | [^\n<] until-3 + until-3-02 ::= [\n] until-3-01 | [/] until-3-03 | [^\n/] until-3 + until-3-03 ::= [\n] until-3-01 | [p] until-3-04 | [^\np] until-3 + until-3-04 ::= [\n] until-3-01 | [a] until-3-05 | [^\na] until-3 + until-3-05 ::= [\n] until-3-01 | [r] until-3-06 | [^\nr] until-3 + until-3-06 ::= [\n] until-3-01 | [a] until-3-07 | [^\na] until-3 + until-3-07 ::= [\n] until-3-01 | [m] until-3-08 | [^\nm] until-3 + until-3-08 ::= [\n] until-3-01 | [e] until-3-09 | [^\ne] until-3 + until-3-09 ::= [\n] until-3-01 | [t] until-3-10 | [^\nt] until-3 + until-3-10 ::= [\n] until-3-01 | [e] until-3-11 | [^\ne] until-3 + until-3-11 ::= [\n] until-3-01 | [r] until-3-12 | [^\nr] until-3 + until-3-12 ::= [\n] until-3-01 | [>] until-3-13 | [^\n>] until-3 + until-3-13 ::= [\n] | [^\n] until-3 )""", gbnf); }); - t.test("ac grammar multiple delimiters", [](testing &t) { + t.test("through grammar multiple delimiters", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { - return p.ac(p.eps(), std::vector{"ab", "cd", "ef"}); + return p.through(p.eps(), { { p.literal("ab") }, { p.literal("cd") }, { p.literal("ef") } }); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { @@ -272,12 +272,238 @@ void test_gbnf_generation(testing &t) { }); assert_gbnf_equal(t, R"""( - ac-1 ::= [a] ac-1-01 | [c] ac-1-03 | [e] ac-1-05 | [^ace] ac-1 - ac-1-01 ::= [b] | [a] ac-1-01 | [c] ac-1-03 | [e] ac-1-05 | [^abce] ac-1 - ac-1-03 ::= [d] | [a] ac-1-01 | [c] ac-1-03 | [e] ac-1-05 | [^acde] ac-1 - ac-1-05 ::= [f] | [a] ac-1-01 | [c] ac-1-03 | [e] ac-1-05 | [^acef] ac-1 - root ::= ac-1 + root ::= until-4 space ::= | " " | "\n"{1,2} [ \t]{0,20} + until-4 ::= [a] until-4-01 | [c] until-4-03 | [e] until-4-05 | [^ace] until-4 + until-4-01 ::= [b] | [a] until-4-01 | [c] until-4-03 | [e] until-4-05 | [^abce] until-4 + until-4-03 ::= [d] | [a] until-4-01 | [c] until-4-03 | [e] until-4-05 | [^acde] until-4 + until-4-05 ::= [f] | [a] until-4-01 | [c] until-4-03 | [e] until-4-05 | [^acef] until-4 + )""", gbnf); + }); + + t.test("until grammar with parser delimiters", [](testing &t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { + return p.until((p.literal("a") + p.literal("b")) | p.literal("cd")); + }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + + assert_gbnf_equal(t, R"""( + root ::= until-5 + space ::= | " " | "\n"{1,2} [ \t]{0,20} + until-5 ::= | [a] until-5-01 | [c] until-5-03 | [^ac] until-5 + until-5-01 ::= | [a] until-5-01 | [c] until-5-03 | [^abc] until-5 + until-5-03 ::= | [a] until-5-01 | [c] until-5-03 | [^acd] until-5 + )""", gbnf); + }); + + t.test("delimiters see through tags and atomics", [](testing &t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { + return p.until(p.tag("open", p.literal("a") + p.atomic(p.literal("b")))); + }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + + assert_gbnf_equal(t, R"""( + root ::= until-5 + space ::= | " " | "\n"{1,2} [ \t]{0,20} + until-5 ::= | [a] until-5-01 | [^a] until-5 + until-5-01 ::= | [a] until-5-01 | [^ab] until-5 + )""", gbnf); + }); + + t.test("delimiters expand a choice in a sequence", [](testing &t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { + return p.until(p.literal("a") + (p.literal("b") | p.literal("d"))); + }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + + assert_gbnf_equal(t, R"""( + root ::= until-5 + space ::= | " " | "\n"{1,2} [ \t]{0,20} + until-5 ::= | [a] until-5-01 | [^a] until-5 + until-5-01 ::= | [a] until-5-01 | [^abd] until-5 + )""", gbnf); + }); + + t.test("delimiters reject anything but sequences, choices, literals, and tokens", [](testing &t) { + bool threw = false; + try { + build_peg_parser([](common_peg_parser_builder & p) { + return p.until(p.literal("a") + p.space()); + }); + } catch (const std::invalid_argument &) { + threw = true; + } + t.assert_equal("space_rejected", true, threw); + }); + + t.test("through branches reject a shared delimiter", [](testing &t) { + bool threw = false; + try { + build_peg_parser([](common_peg_parser_builder & p) { + return p.through(p.eps(), { { p.literal("ab"), p.literal("x") }, { p.literal("ab"), p.literal("y") } }); + }); + } catch (const std::invalid_argument &) { + threw = true; + } + t.assert_equal("duplicate_rejected", true, threw); + }); + + t.test("through grammar branches continue with their rest", [](testing &t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { + return p.through(p.eps(), { { p.literal("ab"), p.literal("x") }, { p.literal("cd") } }); + }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + + assert_gbnf_equal(t, R"""( + root ::= until-4 + space ::= | " " | "\n"{1,2} [ \t]{0,20} + until-4 ::= [a] until-4-01 | [c] until-4-03 | [^ac] until-4 + until-4-01 ::= [b] until-4-rest-0 | [a] until-4-01 | [c] until-4-03 | [^abc] until-4 + until-4-03 ::= [d] | [a] until-4-01 | [c] until-4-03 | [^acd] until-4 + until-4-rest-0 ::= "x" + )""", gbnf); + }); + + t.test("through grammar optional may end before a delimiter", [](testing &t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { + return p.through(p.eps(), { { p.literal("ab"), p.literal("x") } }, true); + }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + + assert_gbnf_equal(t, R"""( + root ::= until-3 + space ::= | " " | "\n"{1,2} [ \t]{0,20} + until-3 ::= | [a] until-3-01 | [^a] until-3 + until-3-01 ::= | [b] until-3-rest-0 | [a] until-3-01 | [^ab] until-3 + until-3-rest-0 ::= "x" + )""", gbnf); + }); + + t.test("trigger rules emit their start and rest when not lazy", [](testing &t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { + auto ab = p.literal("ab"); + auto cd = p.literal("cd"); + auto scan = p.until(ab | cd); + return scan + (p.trigger_rule("one", ab, p.literal("x")) | p.trigger_rule("two", cd, p.literal("y"))); + }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder, false); + }); + + assert_gbnf_equal(t, R"""( + one ::= "ab" "x" + root ::= until-3 (one | two) + space ::= | " " | "\n"{1,2} [ \t]{0,20} + two ::= "cd" "y" + until-3 ::= | [a] until-3-01 | [c] until-3-03 | [^ac] until-3 + until-3-01 ::= | [a] until-3-01 | [c] until-3-03 | [^abc] until-3 + until-3-03 ::= | [a] until-3-01 | [c] until-3-03 | [^acd] until-3 + )""", gbnf); + }); + + t.test("trigger rules make the lazy grammar scan for their starts", [](testing &t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { + auto ab = p.literal("ab"); + auto cd = p.literal("cd"); + auto scan = p.until(ab | cd); + return scan + (p.trigger_rule("one", ab, p.literal("x")) | p.trigger_rule("two", cd, p.literal("y"))); + }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder, true); + }); + + assert_gbnf_equal(t, R"""( + one-rest ::= "x" + root ::= trigger + space ::= | " " | "\n"{1,2} [ \t]{0,20} + trigger ::= | [a] trigger-01 | [c] trigger-03 | [^ac] trigger + trigger-01 ::= | [b] one-rest | [a] trigger-01 | [c] trigger-03 | [^abc] trigger + trigger-03 ::= | [d] two-rest | [a] trigger-01 | [c] trigger-03 | [^acd] trigger + two-rest ::= "y" + )""", gbnf); + }); + + t.test("scanning grammar lets other named tokens through", [](testing &t) { + common_peg_tokens tokens; + tokens.ids.emplace("", 1); + tokens.ids.emplace("", 2); + tokens.tokens.emplace(1, common_peg_token{ "", LLAMA_TOKEN_ATTR_CONTROL }); + tokens.tokens.emplace(2, common_peg_token{ "", LLAMA_TOKEN_ATTR_CONTROL }); + + common_peg_parser_builder p(tokens); + auto start = p.token("") + p.literal("x"); + p.set_root(p.until(start) + p.trigger_rule("tool", start, p.token(""))); + auto parser = p.build(); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder, true); + }); + + assert_gbnf_equal(t, R"""( + root ::= trigger + space ::= | " " | "\n"{1,2} [ \t]{0,20} + tool-rest ::= <[2]> + trigger ::= | <[1]> trigger-01 | (. | <[2]>) trigger + trigger-01 ::= | [x] tool-rest | <[1]> trigger-01 | ([^x] | <[2]>) trigger + )""", gbnf); + }); + + t.test("token sets", [](testing &t) { + common_peg_tokens tokens; + tokens.ids.emplace("", 1); + tokens.ids.emplace("", 2); + tokens.tokens.emplace(1, common_peg_token{ "", LLAMA_TOKEN_ATTR_CONTROL }); + tokens.tokens.emplace(2, common_peg_token{ "", LLAMA_TOKEN_ATTR_CONTROL }); + + common_peg_parser_builder p(tokens); + p.set_root(p.token({ "", "" }) + p.token({ "", "" }, true)); + auto parser = p.build(); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + + assert_gbnf_equal(t, R"""( + root ::= (<[1]> | <[2]>) !<[1-2]> + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf); + }); + + t.test("through grammar suffix delimiter offers both rests", [](testing &t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { + return p.through(p.eps(), { { p.literal("xab"), p.literal("1") }, { p.literal("ab"), p.literal("2") } }); + }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + + assert_gbnf_equal(t, R"""( + root ::= until-5 + space ::= | " " | "\n"{1,2} [ \t]{0,20} + until-5 ::= [x] until-5-01 | [a] until-5-04 | [^ax] until-5 + until-5-01 ::= [x] until-5-01 | [a] until-5-02 | [^ax] until-5 + until-5-02 ::= [b] (until-5-rest-0 | until-5-rest-1) | [x] until-5-01 | [a] until-5-04 | [^abx] until-5 + until-5-04 ::= [b] until-5-rest-1 | [x] until-5-01 | [a] until-5-04 | [^abx] until-5 + until-5-rest-0 ::= "1" + until-5-rest-1 ::= "2" )""", gbnf); }); diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 7b70e7bcc..11fa68f6a 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -305,6 +305,66 @@ static std::string get_expected_description(const llama_grammar_rules & rules, c return result; } +// Every text token shares one ID, the parsers and grammars only tell special tokens apart +static constexpr llama_token FAKE_TEXT_TOKEN = 0; + +struct fake_token { + llama_token id; + std::string piece; +}; + +// Splits text into the special tokens it spells, longest first, and one text token per UTF-8 character. +static std::vector fake_tokenize(const common_peg_tokens & specials, const std::string & text) { + std::vector result; + size_t pos = 0; + while (pos < text.size()) { + llama_token best = LLAMA_TOKEN_NULL; + size_t best_len = 0; + for (const auto & [piece, id] : specials.ids) { + if (piece.size() > best_len && text.compare(pos, piece.size(), piece) == 0) { + best = id; + best_len = piece.size(); + } + } + if (best != LLAMA_TOKEN_NULL) { + result.push_back({ best, text.substr(pos, best_len) }); + pos += best_len; + continue; + } + size_t len = 1; + while (pos + len < text.size() && (static_cast(text[pos + len]) & 0xC0) == 0x80) { + len++; + } + result.push_back({ FAKE_TEXT_TOKEN, text.substr(pos, len) }); + pos += len; + } + return result; +} + +// Returns the tokens from byte offset pos. A token split by pos keeps the rest of its piece as text. +static std::vector fake_tokens_from(const std::vector & tokens, size_t pos) { + std::vector result; + size_t start = 0; + for (const auto & token : tokens) { + const size_t end = start + token.piece.size(); + if (start >= pos) { + result.push_back(token); + } else if (end > pos) { + result.push_back({ FAKE_TEXT_TOKEN, token.piece.substr(pos - start) }); + } + start = end; + } + return result; +} + +static common_chat_input fake_input(const std::vector & tokens) { + common_chat_input input; + for (const auto & token : tokens) { + input.append(token.piece, token.id); + } + return input; +} + // Result of a detailed grammar match attempt struct grammar_match_result { bool success = false; // Did the string fully match the grammar? @@ -318,56 +378,49 @@ struct grammar_match_result { bool incomplete = false; // True if matched all input but grammar expects more }; -// Detailed version of match_string that returns failure information -static grammar_match_result match_string_detailed(const std::string & input, llama_grammar * grammar) { - grammar_match_result result; - result.total_bytes = input.size(); +// Detailed version of match_string that returns failure information, accepting the input token by token +static grammar_match_result match_string_detailed(const std::vector & tokens, llama_grammar * grammar) { + std::string input; + for (const auto & token : tokens) { + input += token.piece; + } - const auto cpts = unicode_cpts_from_utf8(input); - result.total_codepoints = cpts.size(); + grammar_match_result result; + result.total_bytes = input.size(); + result.total_codepoints = unicode_cpts_from_utf8(input).size(); auto & stacks_cur = llama_grammar_get_stacks(grammar); const auto & rules = llama_grammar_get_rules(grammar); size_t byte_pos = 0; + size_t cpt_pos = 0; - for (size_t i = 0; i < cpts.size(); i++) { - const auto & cpt = cpts[i]; - + for (const auto & token : tokens) { // Get expected before accepting (for error reporting) std::string expected_before = get_expected_description(rules, stacks_cur); - llama_grammar_accept(grammar, cpt); - - // Calculate byte position for this codepoint - size_t cpt_bytes = 0; - if (cpt < 0x80) { - cpt_bytes = 1; - } else if (cpt < 0x800) { - cpt_bytes = 2; - } else if (cpt < 0x10000) { - cpt_bytes = 3; - } else { - cpt_bytes = 4; - } - - if (stacks_cur.empty()) { + try { + llama_grammar_accept_token(*grammar, token.id, token.piece); + } catch (const std::runtime_error & /*e*/) { // Grammar failed to match at this point result.matched_bytes = byte_pos; - result.matched_codepoints = i; + result.matched_codepoints = cpt_pos; result.matched_prefix = input.substr(0, byte_pos); - result.failing_char = format_codepoint(cpt); + result.failing_char = token.id == FAKE_TEXT_TOKEN + ? format_codepoint(unicode_cpts_from_utf8(token.piece).at(0)) + : token.piece + " (token " + std::to_string(token.id) + ")"; result.expected_description = expected_before; result.incomplete = false; return result; } - byte_pos += cpt_bytes; + byte_pos += token.piece.size(); + cpt_pos += unicode_cpts_from_utf8(token.piece).size(); } // All input matched - check if grammar is complete result.matched_bytes = input.size(); - result.matched_codepoints = cpts.size(); + result.matched_codepoints = result.total_codepoints; result.matched_prefix = input; if (std::any_of(stacks_cur.begin(), stacks_cur.end(), [](const auto & stack) { return stack.empty(); })) { @@ -1129,6 +1182,7 @@ static std::string g_template_filter; static bool g_force_reconstruction_test = false; static void test_peg_parser(common_chat_templates * tmpls, + const common_peg_tokens & specials, const std::function & init, bool detailed_debug) { // UTF-8-safe truncation helper (same as in test_parser_with_streaming) @@ -1173,8 +1227,15 @@ static void test_peg_parser(common_chat_templates * tmpls, tc.expect.role = "assistant"; } + const bool tokenized = !specials.ids.empty(); + common_chat_session_params session_params; session_params.debug = detailed_debug; + if (tokenized) { + session_params.tokenize = [&](const std::string & text) { + return fake_input(fake_tokenize(specials, text)); + }; + } common_chat_session session(tmpls, nullptr, tc.params, session_params); const auto & parser = session.parser(); common_params_sampling sampling; @@ -1200,14 +1261,27 @@ static void test_peg_parser(common_chat_templates * tmpls, common_chat_msg msg_prev; msg_accum.role = msg_prev.role = "assistant"; - size_t fed = 0; - for (size_t i = 1; i <= tc.input.size(); ++i) { - auto is_partial = i < tc.input.size() || tc.is_partial; - // Use UTF-8 safe truncation to avoid corrupting multi-byte characters - size_t safe_len = utf8_truncate_safe_len(std::string_view(tc.input).substr(0, i)); - common_chat_input chunk(tc.input.substr(fed, safe_len - fed)); - fed = safe_len; - const common_chat_msg & msg_current = is_partial ? session.feed(chunk) : session.finish(chunk); + const auto input_tokens = fake_tokenize(specials, tc.input); + + // A token arrives whole, so a special token is never split across chunks + std::vector chunks; + if (tokenized) { + for (const auto & token : input_tokens) { + chunks.push_back(fake_input({ token })); + } + } else { + size_t fed = 0; + for (size_t i = 1; i <= tc.input.size(); ++i) { + // Use UTF-8 safe truncation to avoid corrupting multi-byte characters + size_t safe_len = utf8_truncate_safe_len(std::string_view(tc.input).substr(0, i)); + chunks.emplace_back(tc.input.substr(fed, safe_len - fed)); + fed = safe_len; + } + } + + for (size_t i = 0; i < chunks.size(); ++i) { + auto is_partial = i + 1 < chunks.size() || tc.is_partial; + const common_chat_msg & msg_current = is_partial ? session.feed(chunks[i]) : session.finish(chunks[i]); for (const auto & diff : common_chat_msg_diff::compute_diffs(msg_prev, msg_current)) { if (!diff.reasoning_content_delta.empty()) { @@ -1222,10 +1296,15 @@ static void test_peg_parser(common_chat_templates * tmpls, while (msg_accum.tool_calls.size() <= diff.tool_call_index) { msg_accum.tool_calls.push_back({ "", "", "" }); } - // Always update name and id from diff (may change during incremental parsing), but only if the delta - // actually contains them + // The name and id are set once the delta carries them. + // A name that is already set must not change, clients concatenate name deltas. if (!diff.tool_call_delta.name.empty()) { - msg_accum.tool_calls[diff.tool_call_index].name = diff.tool_call_delta.name; + auto & name = msg_accum.tool_calls[diff.tool_call_index].name; + if (!name.empty() && name != diff.tool_call_delta.name) { + throw std::runtime_error("Tool call " + std::to_string(diff.tool_call_index) + " name changed from " + + name + " to " + diff.tool_call_delta.name + " during incremental parsing"); + } + name = diff.tool_call_delta.name; } if (!diff.tool_call_delta.id.empty()) { msg_accum.tool_calls[diff.tool_call_index].id = diff.tool_call_delta.id; @@ -1413,9 +1492,10 @@ static void test_peg_parser(common_chat_templates * tmpls, // sampler was already active before model output began — constrain from the // start of the model output (i.e. tc.input). std::string constrained = full_input; + size_t constrain_from = 0; bool grammar_triggered = false; if (earliest_trigger_pos != std::string::npos) { - auto constrain_from = std::max(earliest_trigger_pos, gen_prompt.size()); + constrain_from = std::max(earliest_trigger_pos, gen_prompt.size()); constrained = full_input.substr(constrain_from); grammar_triggered = true; } else if (!sampling.grammar_lazy) { @@ -1425,7 +1505,9 @@ static void test_peg_parser(common_chat_templates * tmpls, // Test the constrained portion against the grammar if (grammar_triggered && !tc.is_partial) { - auto result = match_string_detailed(constrained, grammar.get()); + auto full_tokens = fake_tokenize(specials, gen_prompt); + full_tokens.insert(full_tokens.end(), input_tokens.begin(), input_tokens.end()); + auto result = match_string_detailed(fake_tokens_from(full_tokens, constrain_from), grammar.get()); if (!result.success) { std::string error_msg; if (result.incomplete) { @@ -1514,6 +1596,7 @@ class peg_tester { common_chat_templates_ptr tmpls_; std::string template_path_; bool detailed_debug_; + common_peg_tokens specials_; friend class peg_test_builder; public: @@ -1529,6 +1612,18 @@ class peg_tester { const std::string & template_path() const { return template_path_; } + // Run the tests tokenized with these pieces as special tokens, as if the vocab defined them + peg_tester & specials(const std::vector & pieces) { + llama_token id = FAKE_TEXT_TOKEN + 1; + for (const auto & piece : pieces) { + specials_.ids.emplace(piece, id); + specials_.tokens.emplace(id, common_peg_token{ piece, LLAMA_TOKEN_ATTR_CONTROL }); + id++; + } + common_chat_templates_set_specials(tmpls_.get(), specials_); + return *this; + } + peg_test_builder test(const std::string & input); }; @@ -1649,8 +1744,10 @@ class peg_test_builder { return; } } - LOG_INF("\n\x1b[38;5;126m[%s]\x1b[0m\n%s\n\n", tester_.template_path().c_str(), tc_.input.c_str()); - test_peg_parser(tester_.tmpls_.get(), [this](peg_test_case & t) { t = tc_; }, tester_.detailed_debug_); + LOG_INF("\n\x1b[38;5;126m[%s%s]\x1b[0m\n%s\n\n", tester_.template_path().c_str(), + tester_.specials_.ids.empty() ? "" : ", tokenized", tc_.input.c_str()); + test_peg_parser(tester_.tmpls_.get(), tester_.specials_, [this](peg_test_case & t) { t = tc_; }, + tester_.detailed_debug_); } }; @@ -2269,9 +2366,17 @@ static void test_template_output_peg_parsers(bool detailed_debug) { "const": "42" })"; + // Special tokens of the templates that use the Qwen3-Coder format. + // Check that the vocab of a model defines them before reuse. A model can share the syntax without the tokens. + const std::vector qwen3_coder_specials = { + "<|im_start|>", "<|im_end|>", "", "", + "", "", "", "", + }; + { // Qwen3.5 (basically same as Nemotron, but keeping separate tests just in case) auto tst = peg_tester("models/templates/Qwen3.5-4B.jinja", detailed_debug); + tst.specials(qwen3_coder_specials); tst.test("I'm\nthinking\n\n\nHello, world!\nWhat's up?") .reasoning_format(COMMON_REASONING_FORMAT_AUTO) @@ -2843,6 +2948,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) { { // NVIDIA Nemotron-3 Nano auto tst = peg_tester("models/templates/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.jinja", detailed_debug); + tst.specials(qwen3_coder_specials); tst.test("Hello, world!\nWhat's up?"). enable_thinking(false). @@ -3138,6 +3244,8 @@ static void test_template_output_peg_parsers(bool detailed_debug) { { // Google Gemma 4 (tool calling with Gemma4 dict format) auto tst = peg_tester("models/templates/google-gemma-4-31B-it.jinja"); + tst.specials({ "<|turn>", "", "<|channel>", "", "<|tool_call>", "", + "<|tool_response>", "", "<|tool>", "", "<|\"|>", "<|think|>" }); tst.test("Hello, world!").expect(simple_assist_msg("Hello, world!")).run(); @@ -3766,6 +3874,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) { { // Qwen3-Coder (tool calling with XML-style format) auto tst = peg_tester("models/templates/Qwen3-Coder.jinja", detailed_debug); + tst.specials(qwen3_coder_specials); tst.test("Hello, world!\nWhat's up?").expect(message_assist).expect_reconstruction().run(); @@ -3794,6 +3903,20 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect(message_assist_call) .run(); + // Required tool choice constrains from the first token instead of gating on a trigger + tst.test( + "\n" + "\n" + "\n" + "1\n" + "\n" + "\n" + "") + .tools({ special_function_tool }) + .tool_choice(COMMON_CHAT_TOOL_CHOICE_REQUIRED) + .expect(message_assist_call) + .run(); + tst.test( "Let me call it.\n" "\n" @@ -5895,6 +6018,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) { { auto tst = peg_tester("models/templates/LFM3.jinja", detailed_debug); + tst.specials(qwen3_coder_specials); tst.test("Hello, world!\nWhat's up?") .enable_thinking(false) @@ -7157,6 +7281,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) { // Muse Glimmer format tests { auto tst = peg_tester("models/templates/muse-glimmer.jinja", detailed_debug); + tst.specials({ "<|start|>", "<|message|>", "<|eom|>" }); const std::string call_markup = "\n" @@ -7166,7 +7291,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) { ""; // A plain answer is unaffected - tst.test(" to=user<|message|>Hello, world!\nWhat's up?<|eot|>") + tst.test(" to=user<|message|>Hello, world!\nWhat's up?") .reasoning_format(COMMON_REASONING_FORMAT_AUTO) .expect(message_assist) .run(); @@ -7177,7 +7302,34 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .tools({ special_function_tool }) .reasoning_format(COMMON_REASONING_FORMAT_AUTO) .expect(message_assist_call) - .expect_rules({ "tool-0" }) + .expect_rules({ "tool-0", "tool-calls" }) + .run(); + + // Parallel calls, one message per call + tst.test(" to=special_function<|message|>" + call_markup + "<|eom|>" + "<|start|>assistant to=special_function_with_opt<|message|>" + "\n" + "\n" + "1\n" + "2\n" + "\n" + "") + .tools({ special_function_tool, special_function_tool_with_optional_param }) + .parallel_tool_calls(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .expect_tool_calls({ + { "special_function", R"({"arg1": 1})", {} }, + { "special_function_with_opt", R"({"arg1": 1, "arg2": 2})", {} }, + }) + .run(); + + // Required tool choice after a reasoning message + tst.test(" to=self<|message|>I'm\nthinking<|eom|>" + "<|start|>assistant to=special_function<|message|>" + call_markup) + .tools({ special_function_tool }) + .tool_choice(COMMON_CHAT_TOOL_CHOICE_REQUIRED) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .expect(message_assist_call_thoughts) .run(); // "Inform then act": the model answers the user and calls a tool in ONE generation, @@ -7193,7 +7345,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) { // Markup quoted in an answer has no preceding <|eom|>, so it stays content instead of // becoming an invocation the user never asked for - tst.test(" to=user<|message|>You invoke it like this:\n" + call_markup + "<|eot|>") + tst.test(" to=user<|message|>You invoke it like this:\n" + call_markup) .tools({ special_function_tool }) .reasoning_format(COMMON_REASONING_FORMAT_AUTO) .expect_content("You invoke it like this:\n" + call_markup) @@ -7217,7 +7369,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) { // Tool markup inside the analysis channel is reasoning, not a call tst.test(" to=self<|message|>I could use " + call_markup + " here<|eom|>" - "<|start|>assistant to=user<|message|>Hello!<|eot|>") + "<|start|>assistant to=user<|message|>Hello!") .tools({ special_function_tool }) .reasoning_format(COMMON_REASONING_FORMAT_AUTO) .expect_reasoning("I could use " + call_markup + " here") @@ -7288,6 +7440,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) { // GPT-OSS format tests { auto tst = peg_tester("models/templates/openai-gpt-oss-120b.jinja", detailed_debug); + tst.specials({ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>", "<|return|>", "<|call|>" }); // Basic content only - final channel tst.test("<|channel|>final<|message|>Hello, world!\nWhat's up?").expect(message_assist).run(); @@ -7327,7 +7480,30 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .reasoning_format(COMMON_REASONING_FORMAT_AUTO) .tools({ special_function_tool }) .expect(message_assist_call) - .expect_rules({ "tool-0" }) + .expect_rules({ "tool-0-in-role", "tool-0-in-channel" }) + .run(); + + // A tool whose name prefixes another's must not be reported while the header is still open + tst.test(" to=functions.special_function_with_opt<|channel|>analysis<|message|>{\"arg1\": 1}") + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ special_function_tool, special_function_tool_with_optional_param }) + .expect(simple_assist_msg("", "", "special_function_with_opt", "{\"arg1\": 1}")) + .run(); + + tst.test("<|channel|>commentary to=functions.special_function_with_opt<|message|>{\"arg1\": 1}") + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ special_function_tool, special_function_tool_with_optional_param }) + .expect(simple_assist_msg("", "", "special_function_with_opt", "{\"arg1\": 1}")) + .run(); + + // Required tool choice constrains from the first token instead of gating on a trigger + tst.test( + "<|channel|>analysis<|message|>I'm\nthinking<|end|>" + "<|start|>assistant to=functions.special_function<|channel|>analysis<|message|>{\"arg1\": 1}") + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ special_function_tool }) + .tool_choice(COMMON_CHAT_TOOL_CHOICE_REQUIRED) + .expect(message_assist_call_thoughts) .run(); // Tool call with recipient in channel header: "<|channel|>analysis to=functions.NAME<|message|>JSON" @@ -7544,6 +7720,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) { { auto tst = peg_tester("models/templates/StepFun3.5-Flash.jinja", detailed_debug); + tst.specials(qwen3_coder_specials); tst.test("I was thinking\n\nNow I'm not."). enable_thinking(true). diff --git a/tests/test-grammar-integration.cpp b/tests/test-grammar-integration.cpp index a7a7e968e..ec2b1b784 100644 --- a/tests/test-grammar-integration.cpp +++ b/tests/test-grammar-integration.cpp @@ -41,15 +41,21 @@ struct token_and_piece { std::string piece; }; -// token() encodes a 32-bit ID as 5 bytes: a 0xff marker followed by the ID in big-endian order. -static std::string token(llama_token id) { +// token() encodes a token and its piece as a 0xff marker, the ID in big-endian order, the piece length, and the piece. +static std::string token(llama_token id, const std::string & piece) { return std::string{ static_cast(0xff), static_cast((id >> 24) & 0xff), static_cast((id >> 16) & 0xff), static_cast((id >> 8) & 0xff), - static_cast(id & 0xff) - }; + static_cast(id & 0xff), + static_cast(piece.size()) + } + piece; +} + +// token() without a piece encodes a token whose piece is its <[id]> spelling. +static std::string token(llama_token id) { + return token(id, "<[" + std::to_string(id) + "]>"); } // parse_tokens() parses the token encodes above and UTF-8 text. @@ -60,7 +66,7 @@ static std::vector parse_tokens(const std::string & input) { while (offset < input.size()) { try { if (static_cast(input[offset]) == 0xff) { - if (offset + 5 > input.size()) { + if (offset + 6 > input.size()) { throw std::runtime_error("not enough bytes for token id"); } uint32_t val = @@ -68,9 +74,12 @@ static std::vector parse_tokens(const std::string & input) { (static_cast(input[offset + 2]) << 16) | (static_cast(input[offset + 3]) << 8) | (static_cast(input[offset + 4])); - auto piece = "<[" + std::to_string(val) + "]>"; - result.push_back({static_cast(val), piece}); - offset += 5; + size_t len = static_cast(input[offset + 5]); + if (offset + 6 + len > input.size()) { + throw std::runtime_error("not enough bytes for token piece"); + } + result.push_back({static_cast(val), input.substr(offset + 6, len)}); + offset += 6 + len; } else { uint32_t cpt = unicode_cpt_from_utf8(input, offset); result.push_back({0, unicode_cpt_to_utf8(cpt)}); @@ -84,23 +93,42 @@ static std::vector parse_tokens(const std::string & input) { return result; } +// Returns true if the sampler mask allows the token. The mask rejects a token only when every stack rejects it. +static bool mask_allows(llama_grammar * grammar, const token_and_piece & in) { + const auto decoded = llama_grammar_decode_utf8(in.piece, grammar->partial_utf8); + + llama_grammar_candidates rejects = { { 0, decoded.first.data(), decoded.second, in.token, llama_grammar_is_opaque(*grammar, in.token) } }; + for (const auto & stack : llama_grammar_get_stacks(grammar)) { + rejects = llama_grammar_reject_candidates_for_stack(llama_grammar_get_rules(grammar), stack, rejects); + } + return rejects.empty(); +} + static bool match_string(const std::string & input, llama_grammar * grammar) { const auto parsed = parse_tokens(input); auto & stacks_cur = llama_grammar_get_stacks(grammar); for (const auto & in : parsed) { + const bool allowed = mask_allows(grammar, in); + + // the mask decides what can be sampled, so it has to agree with what accepting the token does try { llama_grammar_accept_token(*grammar, in.token, in.piece); } catch (const std::runtime_error & /*e*/) { - // normally this shouldn't get hit because of llama_grammar_apply - return false; - } + if (allowed) { + fprintf(stderr, "❌ (token %d \"%s\" is allowed by the mask but not accepted)\n", in.token, in.piece.c_str()); + } + assert(!allowed); - if (stacks_cur.empty()) { // no stacks means that the grammar failed to match at this point return false; } + + if (!allowed) { + fprintf(stderr, "❌ (token %d \"%s\" is masked but accepted)\n", in.token, in.piece.c_str()); + } + assert(allowed); } for (const auto & stack : stacks_cur) { @@ -121,6 +149,7 @@ static void test(const std::string & test_desc, const std::string & grammar_str, // Save the original grammar stacks so that we can reset after every new string we want to test const llama_grammar_stacks stacks_org = llama_grammar_get_stacks(grammar); // copy + const llama_partial_utf8 partial_org = grammar->partial_utf8; llama_grammar_stacks & stacks_cur = llama_grammar_get_stacks(grammar); @@ -160,6 +189,7 @@ static void test(const std::string & test_desc, const std::string & grammar_str, // Reset the grammar stacks stacks_cur = stacks_org; + grammar->partial_utf8 = partial_org; } fprintf(stderr, " 🟠 Invalid strings:\n"); @@ -180,6 +210,7 @@ static void test(const std::string & test_desc, const std::string & grammar_str, // Reset the grammar stacks stacks_cur = stacks_org; + grammar->partial_utf8 = partial_org; } // Clean up allocated memory @@ -522,6 +553,137 @@ static void test_simple_grammar() { ); } +static void test_token_rule_utf8() { + // A byte-level token can hold part of a multibyte character, a token rule still takes it as a whole token + test_grammar( + "byte-split characters under token rules", + R"""( + root ::= <[10]> content <[11]> + content ::= (!<[11]>)*)""", + // Passing strings + { + token(10) + token(20, "\xF0") + token(21, "\x9F\xA6\x99") + token(11), + token(10) + token(22, "\xE2") + token(23, "\x98\x80") + token(11), + token(10) + token(24, "a\xF0") + token(21, "\x9F\xA6\x99") + token(11), + }, + // Failing strings + { + token(10) + token(20, "\xF0"), + token(10) + token(20, "\xF0") + token(11), + } + ); + + // A token rule can only start where a token starts, not after char rules consumed part of it + test_grammar( + "token rule after a char rule", + R"""( + root ::= "a" <[11]>)""", + // Passing strings + { + "a" + token(11), + token(12, "a") + token(11), + }, + // Failing strings + { + token(11, "ab"), + token(12, "ab") + token(11), + } + ); + + // A token partly matched by a char rule stays partly matched for every alternative, whichever is checked first + test_grammar( + "token rule alternative after a char rule", + R"""( + root ::= "a" (!<[7]> | "bx"))""", + // Passing strings + { + "abx", + token(12, "abx"), + "a" + token(5), + }, + // Failing strings + { + token(5, "aby"), + token(5, "ab"), + } + ); +} + +static void test_token_sets() { + test_grammar( + "token set with alternatives and ranges", + R"""( + root ::= <[1,5-9,12]>+)""", + // Passing strings + { + token(1), + token(5) + token(9), + token(7) + token(12) + token(1), + }, + // Failing strings + { + token(2), + token(4), + token(10), + token(13), + token(5) + token(11), + } + ); + + test_grammar( + "negated token set", + R"""( + root ::= <[10]> (!<[11,20-29]>)* <[11]>)""", + // Passing strings + { + token(10) + token(11), + token(10) + "text" + token(12) + token(19) + token(30) + token(11), + }, + // Failing strings + { + token(10) + token(20) + token(11), + token(10) + token(29) + token(11), + token(10) + token(11) + token(11), + } + ); +} + +static void test_opaque_tokens() { + // Char rules do not match a token that a token rule names, other tokens still match as text + test_grammar( + "tokens named by token rules are opaque to char rules", + R"""( + root ::= [^x]* <[11]> "done")""", + // Passing strings + { + "abc" + token(11) + "done", + token(11) + "done", + token(12) + token(11) + "done", + }, + // Failing strings + { + "abc" + token(11) + "more" + token(11) + "done", + token(11) + token(11) + "done", + } + ); + + test_grammar( + "tokens named by a negated set are opaque to char rules", + R"""( + root ::= [^x]* !<[11,20-29]>)""", + // Passing strings + { + "abc" + token(30), + token(12) + token(30), + }, + // Failing strings + { + token(25) + token(30), + token(11) + token(30), + } + ); +} + static void test_complex_grammar() { // Test case for a more complex grammar, with both failure strings and success strings test_grammar( @@ -1539,6 +1701,9 @@ int main() { fprintf(stdout, "Running grammar integration tests...\n"); test_simple_grammar(); test_complex_grammar(); + test_token_rule_utf8(); + test_token_sets(); + test_opaque_tokens(); test_special_chars(); test_quantifiers(); test_failure_missing_root(); diff --git a/tests/test-grammar-parser.cpp b/tests/test-grammar-parser.cpp index a6a99520c..6a35fd66c 100644 --- a/tests/test-grammar-parser.cpp +++ b/tests/test-grammar-parser.cpp @@ -179,6 +179,14 @@ int main() verify_failure(deep_grammar.c_str()); } + verify_failure(R"""( + root ::= <[9-5]> + )"""); + + verify_failure(R"""( + root ::= <[4294967296]> + )"""); + verify_parsing(R"""( root ::= "a"{0,5000} )""", { @@ -594,5 +602,21 @@ int main() {LLAMA_GRETYPE_END, 0}, }); + verify_parsing(R"""( + root ::= <[1,5-9,12]> !<[11,20-29]> + )""", { + {"root", 0} + }, { + // root (index 0) + {LLAMA_GRETYPE_TOKEN, 1}, + {LLAMA_GRETYPE_TOKEN_ALT, 5}, + {LLAMA_GRETYPE_TOKEN_RNG_UPPER, 9}, + {LLAMA_GRETYPE_TOKEN_ALT, 12}, + {LLAMA_GRETYPE_TOKEN_NOT, 11}, + {LLAMA_GRETYPE_TOKEN_ALT, 20}, + {LLAMA_GRETYPE_TOKEN_RNG_UPPER, 29}, + {LLAMA_GRETYPE_END, 0}, + }); + return 0; } diff --git a/tests/test-llama-grammar.cpp b/tests/test-llama-grammar.cpp index 25f432a2f..3b3beb438 100644 --- a/tests/test-llama-grammar.cpp +++ b/tests/test-llama-grammar.cpp @@ -202,7 +202,7 @@ int main() uint32_t *cp = new uint32_t[2]; // dynamically allocate memory for code_point cp[0] = 37 + i; cp[1] = 0; - next_candidates[i] = {i, cp, {}, 0}; + next_candidates[i] = {i, cp, {}, 0, false}; } std::vector>> expected_reject = {