chat : implement token aware parsing (#29307)
This commit is contained in:
1 parent
0b51bb2fed
commit
1014775c8a
30 files changed
+1988
-502
No files matched your search
@@ -373,7 +373,9 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
|
||||
auto & p = ctx.p;
|
||||
const auto & inputs = ctx.inputs;
|
||||
|
||||
auto until_suffix = p.rule("until-suffix", p.until(arguments.value_suffix));
|
||||
// A raw string value needs a suffix to end on, without one it is parsed as JSON
|
||||
bool raw_strings = !arguments.value_suffix.empty();
|
||||
auto until_suffix = raw_strings ? p.rule("until-suffix", p.until(arguments.value_suffix)) : p.eps();
|
||||
|
||||
common_peg_parser tool_choice = p.choice();
|
||||
|
||||
@@ -389,9 +391,9 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
|
||||
p.tool_arg(p.tool_arg_open(arguments.name_prefix + p.tool_arg_name(p.literal(param.name)) +
|
||||
arguments.name_suffix) +
|
||||
arguments.value_prefix +
|
||||
(param.schema->may_be_string() ?
|
||||
p.ac(p.tool_arg_string_value(until_suffix) +
|
||||
p.tool_arg_close(p.literal(arguments.value_suffix)), arguments.value_suffix) :
|
||||
(raw_strings && param.schema->may_be_string() ?
|
||||
p.through(p.tool_arg_string_value(until_suffix),
|
||||
p.tool_arg_close(p.literal(arguments.value_suffix))) :
|
||||
(p.tool_arg_json_value(p.schema(
|
||||
p.json(), "tool-" + std::to_string(tool_index) + "-arg-" + std::to_string(param_index) + "-schema", doc, *param.schema)) +
|
||||
p.tool_arg_close(p.literal(arguments.value_suffix)))));
|
||||
|
||||
@@ -71,6 +71,7 @@ struct generation_params {
|
||||
bool is_inference = true;
|
||||
bool add_inference = false;
|
||||
bool mark_input = true; // whether to mark input strings in the jinja context
|
||||
common_peg_tokens specials;
|
||||
|
||||
bool has_continuation() const {
|
||||
return continue_final_message != COMMON_CHAT_CONTINUATION_NONE && !continue_msg.empty();
|
||||
|
||||
@@ -71,6 +71,9 @@ constexpr size_t COMMON_CHAT_MAX_PERMUTE = 6;
|
||||
|
||||
class common_chat_peg_builder : public common_peg_parser_builder {
|
||||
public:
|
||||
common_chat_peg_builder() = default;
|
||||
explicit common_chat_peg_builder(common_peg_tokens tokens) : common_peg_parser_builder(std::move(tokens)) {}
|
||||
|
||||
// Tag constants (from former common_chat_peg_base_builder)
|
||||
static constexpr const char * REASONING_BLOCK = "reasoning-block";
|
||||
static constexpr const char * REASONING = "reasoning";
|
||||
@@ -200,6 +203,14 @@ inline common_peg_arena build_chat_peg_parser(
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
inline common_peg_arena build_chat_peg_parser(
|
||||
const common_peg_tokens & specials,
|
||||
const std::function<common_peg_parser(common_chat_peg_builder & builder)> & fn) {
|
||||
common_chat_peg_builder builder(specials);
|
||||
builder.set_root(fn(builder));
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
class tag_based_peg_mapper {
|
||||
public:
|
||||
std::map<std::string, std::string> tags;
|
||||
|
||||
+18
-2
@@ -315,6 +315,7 @@ struct common_chat_templates {
|
||||
bool add_bos;
|
||||
bool add_eos;
|
||||
bool has_explicit_template; // Model had builtin template or template overridden was specified.
|
||||
common_peg_tokens specials;
|
||||
std::unique_ptr<common_chat_template> template_default; // always set (defaults to chatml)
|
||||
std::unique_ptr<common_chat_template> template_tool_use;
|
||||
std::unique_ptr<common_chat_template> template_tts; // the prompt of a text to speak
|
||||
@@ -802,6 +803,10 @@ void common_chat_templates_free(struct common_chat_templates * tmpls) {
|
||||
delete tmpls;
|
||||
}
|
||||
|
||||
void common_chat_templates_set_specials(struct common_chat_templates * tmpls, common_peg_tokens specials) {
|
||||
tmpls->specials = std::move(specials);
|
||||
}
|
||||
|
||||
bool common_chat_templates_was_explicit(const struct common_chat_templates * tmpls) {
|
||||
return tmpls->has_explicit_template;
|
||||
}
|
||||
@@ -978,6 +983,7 @@ common_chat_templates_ptr common_chat_templates_init(const struct llama_model *
|
||||
std::string token_eos = eos_token_override;
|
||||
bool add_bos = false;
|
||||
bool add_eos = false;
|
||||
common_peg_tokens specials;
|
||||
if (model) {
|
||||
const auto * vocab = llama_model_get_vocab(model);
|
||||
const auto get_token = [&](llama_token token, const char * name, const char * jinja_variable_name) {
|
||||
@@ -997,11 +1003,13 @@ common_chat_templates_ptr common_chat_templates_init(const struct llama_model *
|
||||
token_eos = get_token(llama_vocab_eos(vocab), "EOS", "eos_token");
|
||||
add_bos = llama_vocab_get_add_bos(vocab);
|
||||
add_eos = llama_vocab_get_add_eos(vocab);
|
||||
specials = common_peg_tokens(vocab);
|
||||
}
|
||||
common_chat_templates_ptr tmpls(new common_chat_templates());
|
||||
tmpls->has_explicit_template = has_explicit_template;
|
||||
tmpls->add_bos = add_bos;
|
||||
tmpls->add_eos = add_eos;
|
||||
tmpls->specials = std::move(specials);
|
||||
try {
|
||||
tmpls->template_default = std::make_unique<common_chat_template>(default_template_src, token_bos, token_eos);
|
||||
} catch (const std::exception & e) {
|
||||
@@ -1543,6 +1551,7 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
|
||||
params.add_generation_prompt = inputs.add_generation_prompt;
|
||||
params.add_bos = tmpls->add_bos;
|
||||
params.add_eos = tmpls->add_eos;
|
||||
params.specials = tmpls->specials;
|
||||
|
||||
params.continue_final_message = inputs.continue_final_message;
|
||||
if (params.continue_final_message != COMMON_CHAT_CONTINUATION_NONE) {
|
||||
@@ -1825,6 +1834,7 @@ common_chat_input common_chat_input_tokenize(const llama_vocab * vocab, const st
|
||||
}
|
||||
if (input.text != text) {
|
||||
// the pieces do not give back the same text, keep the text without tokens
|
||||
LOG_WRN("%s: token pieces do not match the text, parsing without token ids\n", __func__);
|
||||
return common_chat_input(text);
|
||||
}
|
||||
return input;
|
||||
@@ -1935,11 +1945,17 @@ common_chat_session::common_chat_session(const common_chat_templates * tm
|
||||
thinking_ends = std::move(applied.thinking_end_tags);
|
||||
|
||||
parser_params.format = applied.format;
|
||||
parser_params.generation_prompt = vocab ? common_chat_input_tokenize(vocab, applied.generation_prompt)
|
||||
: common_chat_input(applied.generation_prompt);
|
||||
parser_params.debug = params.debug;
|
||||
parser_params.parser = std::move(applied.parser);
|
||||
|
||||
if (vocab) {
|
||||
parser_params.generation_prompt = common_chat_input_tokenize(vocab, applied.generation_prompt);
|
||||
} else if (params.tokenize) {
|
||||
parser_params.generation_prompt = params.tokenize(applied.generation_prompt);
|
||||
} else {
|
||||
parser_params.generation_prompt = common_chat_input(applied.generation_prompt);
|
||||
}
|
||||
|
||||
delimiters = std::move(applied.message_delimiters);
|
||||
|
||||
if (vocab) {
|
||||
|
||||
@@ -372,6 +372,9 @@ common_chat_templates_ptr common_chat_templates_init(const struct llama_model *
|
||||
const std::string & bos_token_override = "",
|
||||
const std::string & eos_token_override = "");
|
||||
|
||||
// override special tokens for testing
|
||||
void common_chat_templates_set_specials(struct common_chat_templates * tmpls, common_peg_tokens specials);
|
||||
|
||||
bool common_chat_templates_was_explicit(const struct common_chat_templates * tmpls);
|
||||
std::string common_chat_templates_source(const struct common_chat_templates * tmpls, const std::string & variant = "");
|
||||
|
||||
@@ -397,6 +400,9 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, const
|
||||
struct common_chat_session_params {
|
||||
bool echo = false; // include the assistant prefill in the output when continuing a message
|
||||
bool debug = false; // enable debug output for the PEG parser
|
||||
|
||||
// tokenize the generation prompt when there is no vocab for testing
|
||||
std::function<common_chat_input(const std::string &)> tokenize;
|
||||
};
|
||||
|
||||
class common_chat_session {
|
||||
|
||||
@@ -676,8 +676,8 @@ private:
|
||||
const auto & node = trie.nodes[idx];
|
||||
std::string rejects;
|
||||
auto first = true;
|
||||
for (const auto & [cpt, child] : node.children) {
|
||||
std::string c = common_unicode_cpt_to_utf8(cpt);
|
||||
for (const auto & [sym, child] : node.children) {
|
||||
std::string c = common_unicode_cpt_to_utf8(sym.value);
|
||||
rejects += c;
|
||||
if (first) {
|
||||
first = false;
|
||||
|
||||
+29
-26
@@ -24,7 +24,9 @@ struct depth_guard {
|
||||
// Gemma 4 value parsers
|
||||
void gemma4_value_rules(common_chat_peg_builder & p) {
|
||||
// The tags are what gemma4_to_json() in the mapper reads, the rules may wrap them freely.
|
||||
p.rule("gemma4-string", p.tag(gemma4::STRING, p.literal("<|\"|>") + p.tag(gemma4::STRING_CONTENT, p.until("<|\"|>")) + p.literal("<|\"|>")));
|
||||
auto string_delim = p.token("<|\"|>");
|
||||
|
||||
p.rule("gemma4-string", p.tag(gemma4::STRING, string_delim + p.tag(gemma4::STRING_CONTENT, p.until(string_delim)) + string_delim));
|
||||
|
||||
p.rule("gemma4-bool", p.tag(gemma4::BOOL, p.json_bool()));
|
||||
|
||||
@@ -34,7 +36,7 @@ void gemma4_value_rules(common_chat_peg_builder & p) {
|
||||
|
||||
// Keys are bare in the template, but Gemma 4 occasionally wraps them in quotes
|
||||
p.rule("gemma4-dict-key", p.tag(gemma4::DICT_KEY, p.choice({
|
||||
p.literal("<|\"|>") + p.tag(gemma4::DICT_KEY_NAME, p.until("<|\"|>")) + p.literal("<|\"|>"),
|
||||
string_delim + p.tag(gemma4::DICT_KEY_NAME, p.until(string_delim)) + string_delim,
|
||||
p.literal("\"") + p.tag(gemma4::DICT_KEY_NAME, p.chars("[^\"]", 1, -1)) + p.literal("\""),
|
||||
p.literal("'") + p.tag(gemma4::DICT_KEY_NAME, p.chars("[^']", 1, -1)) + p.literal("'"),
|
||||
p.tag(gemma4::DICT_KEY_NAME, p.chars("[^:}]", 1, -1)),
|
||||
@@ -144,7 +146,7 @@ class gemma4_schema_parser {
|
||||
alts |= name();
|
||||
}
|
||||
// Gemma 4 may emit keys in its string delimiters or quotes
|
||||
alts |= p_.literal("<|\"|>") + name() + p_.literal("<|\"|>");
|
||||
alts |= p_.token("<|\"|>") + name() + p_.token("<|\"|>");
|
||||
if (key.find('\'') == std::string::npos) {
|
||||
alts |= p_.literal("'") + name() + p_.literal("'");
|
||||
}
|
||||
@@ -331,18 +333,23 @@ common_chat_params common_chat_params_init_gemma4(const common_chat_template &
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto start = p.rule("start", p.optional(p.literal("<|turn>model\n")));
|
||||
data.parser = build_chat_peg_parser(inputs.specials, [&](common_chat_peg_builder & p) {
|
||||
auto start = p.rule("start", p.optional(p.token("<|turn>") + p.literal("model\n")));
|
||||
auto channel_start = p.token("<|channel>");
|
||||
auto channel_end = p.token("<channel|>");
|
||||
auto think_start = channel_start + p.literal("thought");
|
||||
auto tool_call_start = p.token("<|tool_call>");
|
||||
// the model may start a tool call before closing the thought channel
|
||||
auto think_end = channel_end | p.peek(tool_call_start);
|
||||
|
||||
if (extract_reasoning) {
|
||||
// the model may start a tool call before closing the thought channel
|
||||
p.rule("thought", p.literal("<|channel>thought") + p.space() + p.reasoning(p.until_one_of({"<channel|>", "<|tool_call>"})) + (p.literal("<channel|>") | p.peek(p.literal("<|tool_call>"))));
|
||||
p.rule("thought", think_start + p.space() + p.reasoning(p.until(channel_end | tool_call_start)) + think_end);
|
||||
} else {
|
||||
p.rule("thought", p.content(p.literal("<|channel>thought") + p.space() + p.until_one_of({"<channel|>", "<|tool_call>"}) + (p.literal("<channel|>") | p.peek(p.literal("<|tool_call>")))));
|
||||
p.rule("thought", p.content(think_start + p.space() + p.until(channel_end | tool_call_start) + think_end));
|
||||
}
|
||||
|
||||
auto consume_empty_channels = p.gbnf(p.zero_or_more(p.literal("<|channel>") + p.negate(p.literal("thought"))), "");
|
||||
auto thought = (p.peek(p.literal("<|channel>")) + consume_empty_channels + p.ref("thought")) | p.negate(p.literal("<|channel>"));
|
||||
auto consume_empty_channels = p.gbnf(p.zero_or_more(channel_start + p.negate(p.literal("thought"))), "");
|
||||
auto thought = (p.peek(channel_start) + consume_empty_channels + p.ref("thought")) | p.negate(channel_start);
|
||||
|
||||
if (has_response_format) {
|
||||
auto response_format = p.literal("```json") <<
|
||||
@@ -378,39 +385,35 @@ common_chat_params common_chat_params_init_gemma4(const common_chat_template &
|
||||
})));
|
||||
});
|
||||
|
||||
auto tool_call = p.trigger_rule("tool-call", p.repeat(
|
||||
"<|tool_call>call:" + tool_choice + "<tool_call|>",
|
||||
/* min = */ inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0,
|
||||
/* max = */ inputs.parallel_tool_calls ? -1 : 1
|
||||
));
|
||||
auto tool_call_rest = p.literal("call:") + tool_choice + p.token("<tool_call|>");
|
||||
auto tool_call = p.rule("tool-call", tool_call_start + tool_call_rest);
|
||||
auto more = inputs.parallel_tool_calls ? p.zero_or_more(tool_call) : p.eps();
|
||||
|
||||
auto tool_calls = p.trigger_rule("tool-calls", tool_call_start, tool_call_rest + more);
|
||||
|
||||
if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
|
||||
return start + thought + tool_call;
|
||||
return start + thought + tool_calls;
|
||||
}
|
||||
|
||||
auto scan_to_toolcall = p.rule("scan-to-toolcall", p.until("<|tool_call>"));
|
||||
auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", "<channel|>", "<|tool_call>"})));
|
||||
auto scan_to_toolcall = p.rule("scan-to-toolcall", p.until(tool_call_start));
|
||||
auto content = p.rule("content", p.content(p.until(channel_start | channel_end | tool_call_start)));
|
||||
auto message = p.rule("message", thought + content);
|
||||
return start + p.zero_or_more(message) + scan_to_toolcall + tool_call;
|
||||
return start + p.zero_or_more(message) + scan_to_toolcall + p.optional(tool_calls);
|
||||
}
|
||||
|
||||
// Gemma 4 may emit an extra <|channel>thought\n<channel|> at the end of the content. It may
|
||||
// also emit a single trailing <channel|> token. Consume all complete reasoning blocks and
|
||||
// then stop at the first unmatched <channel|> token.
|
||||
auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", "<channel|>"})));
|
||||
auto content = p.rule("content", p.content(p.until(channel_start | channel_end)));
|
||||
auto message = p.rule("message", thought + content);
|
||||
return start + p.one_or_more(message);
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
||||
data.grammar_lazy = false;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
data.parser.build_grammar(builder, !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)));
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<|tool_call>" },
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
|
||||
+31
-30
@@ -65,17 +65,18 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto start = p.rule("start", p.literal("<|start|>assistant"));
|
||||
auto end = p.rule("end", p.literal("<|end|>"));
|
||||
auto content = p.rule("message-content", p.until("<|end|>"));
|
||||
auto channel = p.literal("<|channel|>") + (p.literal("commentary") | p.literal("analysis"));
|
||||
data.parser = build_chat_peg_parser(inputs.specials, [&](common_chat_peg_builder & p) {
|
||||
auto start_assistant = p.token("<|start|>") + p.literal("assistant");
|
||||
auto start = p.rule("start", start_assistant);
|
||||
auto end = p.rule("end", p.token("<|end|>"));
|
||||
auto content = p.rule("message-content", p.until(p.token("<|end|>")));
|
||||
auto channel = p.token("<|channel|>") + (p.literal("commentary") | p.literal("analysis"));
|
||||
auto constrain_type = p.chars("[A-Za-z0-9_-]", 1, -1);
|
||||
auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);
|
||||
auto constraint = p.optional(p.chars("[ ]", 1, -1) + p.optional(p.token("<|constrain|>")) + constrain_type);
|
||||
|
||||
// Occasionally, gpt-oss-20b will prefix channels with this commentary
|
||||
auto stray_commentary = p.optional(p.literal("<|channel|>commentary") + p.optional(p.literal(" to=assistant")));
|
||||
auto start_analysis = stray_commentary + p.literal("<|channel|>analysis<|message|>");
|
||||
auto stray_commentary = p.optional(p.token("<|channel|>") + p.literal("commentary") + p.optional(p.literal(" to=assistant")));
|
||||
auto start_analysis = stray_commentary + p.token("<|channel|>") + p.literal("analysis") + p.token("<|message|>");
|
||||
|
||||
if (extract_reasoning) {
|
||||
p.rule("analysis", start_analysis + p.reasoning(content) + end);
|
||||
@@ -84,8 +85,8 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
|
||||
}
|
||||
|
||||
auto analysis = p.ref("analysis");
|
||||
auto preamble = p.rule("preamble", p.literal("<|channel|>commentary<|message|>") + p.content(content) + end);
|
||||
auto final_msg = p.rule("final", stray_commentary + p.literal("<|channel|>final") + constraint + p.literal("<|message|>") + p.content(content));
|
||||
auto preamble = p.rule("preamble", p.token("<|channel|>") + p.literal("commentary") + p.token("<|message|>") + p.content(content) + end);
|
||||
auto final_msg = p.rule("final", stray_commentary + p.token("<|channel|>") + p.literal("final") + constraint + p.token("<|message|>") + p.content(content));
|
||||
|
||||
// Consume any unsolicited tool calls, e.g. builtin functions
|
||||
auto unsolicited = p.rule("unsolicited", p.atomic(p.optional(channel) + p.literal(" to=") + content + end));
|
||||
@@ -94,58 +95,58 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
|
||||
|
||||
if (has_response_format) {
|
||||
auto response_format = p.rule("response-format",
|
||||
p.literal("<|channel|>final") + constraint + p.literal("<|message|>") +
|
||||
p.token("<|channel|>") + p.literal("final") + constraint + p.token("<|message|>") +
|
||||
p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)));
|
||||
|
||||
return p.zero_or_more(start + analysis) + start + response_format;
|
||||
}
|
||||
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
auto tool_choice = p.choice();
|
||||
auto in_role = p.choice();
|
||||
auto in_channel = p.choice();
|
||||
|
||||
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
const auto params = common_chat_tool_parameters(function);
|
||||
|
||||
auto func_name = p.literal(" to=functions.") + p.tool_name(p.literal(name));
|
||||
auto func_name = p.tool_name(p.literal(name));
|
||||
auto args = p.tool_args(p.schema(p.json(), "tool-" + std::to_string(tool_index) + "-schema", params));
|
||||
|
||||
// recipient in role header
|
||||
// <|start|>assistant to=functions.NAME<|channel|>(commentary|analysis)[constraint]<|message|>ARGS
|
||||
auto tool_in_role = p.tool(p.tool_open(func_name + channel + constraint + p.literal("<|message|>")) + args);
|
||||
in_role |= p.rule("tool-" + std::to_string(tool_index) + "-in-role",
|
||||
p.atomic(func_name + channel + constraint + p.token("<|message|>")) + args);
|
||||
|
||||
// recipient in channel header
|
||||
// <|channel|>(commentary|analysis) to=functions.NAME[constraint]<|message|>ARGS
|
||||
auto tool_in_channel = p.tool(p.tool_open(channel + func_name + constraint + p.literal("<|message|>")) + args);
|
||||
|
||||
tool_choice |= p.rule("tool-" + std::to_string(tool_index), tool_in_role | tool_in_channel);
|
||||
// <|start|>assistant<|channel|>(commentary|analysis) to=functions.NAME[constraint]<|message|>ARGS
|
||||
in_channel |= p.rule("tool-" + std::to_string(tool_index) + "-in-channel",
|
||||
p.atomic(func_name + constraint + p.token("<|message|>")) + args);
|
||||
});
|
||||
|
||||
auto tool_call = p.trigger_rule("tool-call", tool_choice);
|
||||
// The recipient prefix is the trigger, so the grammar constrains the name that follows it
|
||||
auto to_functions = p.tool_open(p.literal(" to=functions."));
|
||||
auto tool_call = p.choice({
|
||||
p.trigger_rule("tool-call-in-role", start_assistant + to_functions, in_role),
|
||||
p.trigger_rule("tool-call-in-channel", start_assistant + channel + to_functions, in_channel)
|
||||
});
|
||||
|
||||
if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
|
||||
return p.zero_or_more(start + any) + start + tool_call;
|
||||
return p.zero_or_more(start + any) + tool_call;
|
||||
}
|
||||
|
||||
return p.zero_or_more(start + any) + start + (tool_call | final_msg);
|
||||
return p.zero_or_more(start + any) + (tool_call | (start + final_msg));
|
||||
}
|
||||
|
||||
return p.zero_or_more(start + any) + start + (final_msg | unsolicited);
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
||||
// The grammar scans for the triggers itself, so the sampler runs it from the first token
|
||||
data.grammar_lazy = false;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
data.parser.build_grammar(builder, !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^\\s+to$" },
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^<\\|channel\\|>(?:commentary|analysis)\\s+to=functions$" },
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(\\s+to)" },
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(<\\|channel\\|>(?:commentary|analysis)\\s+to)" }
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
|
||||
@@ -99,8 +99,11 @@ common_chat_params common_chat_params_init_harmony_xml(const common_chat_templat
|
||||
auto arg_close = p.tool_arg_close(p.gbnf(
|
||||
p.literal(PARAM_CLOSE) | (p.literal("\n") + p.peek(p.optional(p.literal("\n")) + next_open)),
|
||||
gbnf_format_literal("\n") + " " + gbnf_format_literal(PARAM_CLOSE.substr(1)) + "?"));
|
||||
auto arg_string = p.rule("rfl-arg-string",
|
||||
p.ac(p.tool_arg_string_value(p.until_one_of(string_ends)) + arg_close, PARAM_CLOSE));
|
||||
auto string_value = p.tool_arg_string_value(p.until_one_of(string_ends));
|
||||
auto arg_string = p.rule("rfl-arg-string", p.through(string_value, p.tool_arg_close(p.literal(PARAM_CLOSE))));
|
||||
// the parser also accepts a value with no closing tag, the grammar still requires the closing tag
|
||||
auto missing_close = string_value + p.tool_arg_close(p.literal("\n") + p.peek(p.optional(p.literal("\n")) + next_open));
|
||||
arg_string = p.gbnf(arg_string | missing_close, "rfl-arg-string");
|
||||
auto arg_sep = p.optional(p.literal("\n"));
|
||||
|
||||
std::string call_close_gbnf = gbnf_format_literal("<|end|>");
|
||||
|
||||
@@ -91,7 +91,7 @@ common_chat_params common_chat_params_init_k2_horizon(const common_chat_template
|
||||
}
|
||||
auto think_body = p.until_one_of(data.thinking_end_tags);
|
||||
auto think_block = [&](const common_peg_parser & body) {
|
||||
return p.optional(THINK_START + p.space() + p.ac(body + think_end, data.thinking_end_tags));
|
||||
return p.optional(THINK_START + p.space() + p.through(body, think_end));
|
||||
};
|
||||
auto reasoning = extract_reasoning ? think_block(p.reasoning(think_body)) : p.eps();
|
||||
|
||||
@@ -110,7 +110,7 @@ common_chat_params common_chat_params_init_k2_horizon(const common_chat_template
|
||||
tool_choice = p.standard_json_tools(CALL_START, CALL_END, inputs.tools, false, true);
|
||||
} else {
|
||||
auto arg_close = p.tool_arg_close(p.literal(ARG_VAL_END));
|
||||
auto arg_string = p.rule("xml-arg-string", p.ac(p.tool_arg_string_value(p.until(ARG_VAL_END)) + arg_close, ARG_VAL_END));
|
||||
auto arg_string = p.rule("xml-arg-string", p.through(p.tool_arg_string_value(p.until(ARG_VAL_END)), arg_close));
|
||||
|
||||
// The models leave out <ifm|arg_type> even when asked for xml_typed
|
||||
auto arg_type = call_format == "xml_typed" ? p.optional(ARG_TYPE + p.until(ARG_TYPE_END) + ARG_TYPE_END + p.space()) : p.eps();
|
||||
|
||||
@@ -63,8 +63,8 @@ common_chat_params common_chat_params_init_minicpm5(const common_chat_template &
|
||||
// CDATA lets a value carry characters that would otherwise close the tag (e.g.
|
||||
// </param>); capture the inner text only, excluding the CDATA markers.
|
||||
auto string_value = p.choice({
|
||||
p.literal("<![CDATA[") + p.ac(p.tool_arg_string_value(p.until("]]>")) + p.literal("]]>"), "]]>") + p.tool_arg_close(p.literal("</param>")),
|
||||
p.negate(p.literal("<![CDATA[")) + p.ac(p.tool_arg_string_value(p.until("</param>")) + p.tool_arg_close(p.literal("</param>")), "</param>")
|
||||
p.literal("<![CDATA[") + p.through(p.tool_arg_string_value(p.until("]]>")), "]]>") + p.tool_arg_close(p.literal("</param>")),
|
||||
p.negate(p.literal("< {
|
||||
auto start = p.rule("start", p.literal("<|start|>assistant"));
|
||||
data.parser = build_chat_peg_parser(inputs.specials, [&](common_chat_peg_builder & p) {
|
||||
auto start_token = p.token("<|start|>");
|
||||
auto message = p.token("<|message|>");
|
||||
auto eom = p.token("<|eom|>");
|
||||
auto start = p.rule("start", start_token + p.literal("assistant"));
|
||||
|
||||
if (!extract_reasoning && !include_grammar) {
|
||||
return start + p.content(p.rest());
|
||||
}
|
||||
|
||||
if (extract_reasoning) {
|
||||
p.rule("analysis", p.literal(" to=self<|message|>") + p.reasoning(p.until("<|eom|>")) + p.literal("<|eom|>"));
|
||||
p.rule("analysis", p.literal(" to=self") + message + p.reasoning(p.until(eom)) + eom);
|
||||
} else {
|
||||
p.rule("analysis", p.literal(" to=self<|message|>") + p.content(p.until("<|eom|>")) + p.literal("<|eom|>"));
|
||||
p.rule("analysis", p.literal(" to=self") + message + p.content(p.until(eom)) + eom);
|
||||
}
|
||||
auto analysis = p.ref("analysis");
|
||||
|
||||
auto recipient = p.optional(p.literal(" to=user"));
|
||||
auto final_msg = p.rule("final", recipient + p.literal("<|message|>") +
|
||||
p.content(p.until_one_of({ "<|eot|>", "<|eom|>" })));
|
||||
auto final_msg = p.rule("final", recipient + message + p.content(p.until(eom)));
|
||||
|
||||
if (has_response_format) {
|
||||
auto response_json = p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema));
|
||||
auto response_format = p.rule("response-format",
|
||||
recipient + p.literal("<|message|>") +
|
||||
recipient + message +
|
||||
((p.literal("```json") + p.space() + response_json + p.space() + p.literal("```")) | response_json));
|
||||
|
||||
return p.zero_or_more(start + analysis) + start + response_format;
|
||||
}
|
||||
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
auto string_value = p.ac(
|
||||
p.tool_arg_string_value(p.until("</atem:parameter>")) + p.tool_arg_close(p.literal("</atem:parameter>")),
|
||||
"</atem:parameter>");
|
||||
auto string_value = p.through(p.tool_arg_string_value(p.until("</atem:parameter>")),
|
||||
p.tool_arg_close(p.literal("</atem:parameter>")));
|
||||
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
|
||||
@@ -106,40 +107,35 @@ common_chat_params common_chat_params_init_muse_glimmer(const common_chat_templa
|
||||
args = p.zero_or_more(p.choice(arg_rules) + p.space());
|
||||
}
|
||||
|
||||
auto tool_parser = p.tool(
|
||||
p.tool_open(p.literal(" to=") + p.until("<|message|>") +
|
||||
p.literal("<|message|><atem:function_calls>") + p.space() +
|
||||
p.literal("<atem:invoke name=\"") + p.tool_name(p.literal(name)) + p.literal("\">") + p.space())
|
||||
auto open = p.tool_open(start_token + p.literal("assistant to=") + p.tool_name(p.literal(name)) + message);
|
||||
auto body = p.rule("tool-" + std::to_string(tool_index),
|
||||
p.literal("<atem:function_calls>") + p.space() + p.literal("<atem:invoke name=\"" + name + "\">") + p.space()
|
||||
<< p.tool_args(args)
|
||||
<< p.tool_close(p.literal("</atem:invoke>") + p.space() + p.literal("</atem:function_calls>")));
|
||||
|
||||
tool_choice |= p.rule("tool-" + std::to_string(tool_index), tool_parser);
|
||||
tool_choice |= open + body;
|
||||
});
|
||||
|
||||
auto tool_calls = inputs.parallel_tool_calls
|
||||
? p.trigger_rule("tool-call", tool_choice + p.zero_or_more(p.literal("<|eom|>") + start + tool_choice))
|
||||
: p.trigger_rule("tool-call", tool_choice);
|
||||
|
||||
auto more = inputs.parallel_tool_calls ? p.zero_or_more(eom + tool_choice) : p.eps();
|
||||
auto tool_calls = p.rule("tool-calls", tool_choice + more);
|
||||
|
||||
if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
|
||||
return p.zero_or_more(start + analysis) + start + tool_calls;
|
||||
return p.zero_or_more(start + analysis) + tool_calls;
|
||||
}
|
||||
auto trailing_calls = p.optional(p.literal("<|eom|>") + start + tool_calls);
|
||||
return p.zero_or_more(start + analysis) + start + (tool_calls | (final_msg + trailing_calls));
|
||||
auto trailing_calls = p.optional(eom + tool_calls);
|
||||
return p.zero_or_more(start + analysis) + (tool_calls | (start + final_msg + trailing_calls));
|
||||
}
|
||||
|
||||
return p.zero_or_more(start + analysis) + start + final_msg;
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
||||
// Every message starts with a recipient header, so the grammar is not lazy.
|
||||
// The model cannot generate a recipient that is not self, user, or a tool.
|
||||
data.grammar_lazy = false;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
data.parser.build_grammar(builder);
|
||||
});
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN,
|
||||
"(?:^|<\\|start\\|>assistant)( to=(?!self<\\|message\\|>)(?!user<\\|message\\|>)[^<]*?<\\|message\\|>)" },
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
|
||||
@@ -86,9 +86,8 @@ common_chat_params common_chat_params_init_muse_spark(const common_chat_template
|
||||
}
|
||||
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
auto string_value = p.ac(
|
||||
p.tool_arg_string_value(p.until("</atem:parameter>")) + p.tool_arg_close(p.literal("</atem:parameter>")),
|
||||
"</atem:parameter>");
|
||||
auto string_value = p.through(p.tool_arg_string_value(p.until("</atem:parameter>")),
|
||||
p.tool_arg_close(p.literal("</atem:parameter>")));
|
||||
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
|
||||
|
||||
@@ -59,27 +59,16 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
std::vector<std::string> tool_call_starts = { "<tool_call>" };
|
||||
|
||||
if (is_qwen3_coder) {
|
||||
// Match complete <function=name> opener for Qwen3-Coder models that occasionally omit the
|
||||
// starting <tool_call>. The model may hallucinate a tool name, but it is preferable over
|
||||
// constraining on <function which may occur in valid content generation, e.g. #include <functional>
|
||||
foreach_function(inputs.tools, [&](size_t, const json & tool) {
|
||||
const std::string name = tool.at("function").at("name");
|
||||
tool_call_starts.push_back("<function=" + name + ">");
|
||||
});
|
||||
}
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
data.parser = build_chat_peg_parser(inputs.specials, [&](common_chat_peg_builder & p) {
|
||||
auto generation_prompt = p.literal(GEN_PREFIX);
|
||||
|
||||
// The template strips the newline before </think>, so leave it out of the reasoning
|
||||
auto think_body = p.until_one_of({ "\n</think>", "</think>", "<tool_call>" });
|
||||
auto think_close = p.literal("\n</think>") | p.literal("</think>") | p.peek(p.literal("<tool_call>"));
|
||||
auto think_end = (p.literal("\n") + p.token("</think>")) | p.token("</think>");
|
||||
auto think_body = p.until(think_end | p.token("<tool_call>"));
|
||||
auto think_close = think_end | p.peek(p.token("<tool_call>"));
|
||||
|
||||
auto reasoning_extracted = p.optional("<think>" + p.space() + p.reasoning(think_body) + think_close);
|
||||
auto reasoning_skipped = p.optional("<think>" + p.space() + think_body + think_close);
|
||||
auto reasoning_extracted = p.optional(p.token("<think>") + p.space() + p.reasoning(think_body) + think_close);
|
||||
auto reasoning_skipped = p.optional(p.token("<think>") + p.space() + think_body + think_close);
|
||||
|
||||
auto reasoning = p.eps();
|
||||
if (supports_reasoning && extract_reasoning) {
|
||||
@@ -99,7 +88,13 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
auto arg_close = p.tool_arg_close(p.literal("\n</parameter>\n"));
|
||||
auto arg_string = p.rule("xml-arg-string",
|
||||
p.ac(p.tool_arg_string_value(p.until("\n</parameter>\n")) + arg_close, "\n</parameter>\n"));
|
||||
p.through(p.tool_arg_string_value(p.until("\n</parameter>\n")), arg_close));
|
||||
|
||||
auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;
|
||||
|
||||
auto more = inputs.parallel_tool_calls ? p.zero_or_more(p.ref("tool-call")) : p.eps();
|
||||
auto starts = p.token("<tool_call>");
|
||||
auto tool_calls = p.choice();
|
||||
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
|
||||
@@ -155,35 +150,29 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat
|
||||
args = args + p.zero_or_more(p.choice(optional_args));
|
||||
}
|
||||
|
||||
auto func = p.tool(p.tool_open("<function=" + p.tool_name(p.literal(name)) + ">\n") +
|
||||
p.tool_args(args) +
|
||||
p.tool_close(p.literal("</function>\n")));
|
||||
auto opener = p.tool_open("<function=" + p.tool_name(p.literal(name)) + ">");
|
||||
auto body = p.literal("\n") + p.tool_args(args) + p.tool_close(p.literal("</function>\n"));
|
||||
|
||||
tool_choice |= p.rule("tool-" + std::to_string(tool_index), func);
|
||||
tool_choice |= p.rule("tool-" + std::to_string(tool_index), p.tool(opener + body));
|
||||
|
||||
if (is_qwen3_coder) {
|
||||
// Qwen3-Coder models sometimes omit <tool_call>, so the full <function=name> opener is also a trigger.
|
||||
// Do not trigger on <function alone, it occurs in normal content, e.g. #include <functional>.
|
||||
tool_calls |= p.trigger_rule("tool-calls-" + std::to_string(tool_index), opener, body + p.token("</tool_call>") + p.space() + more);
|
||||
starts |= opener;
|
||||
}
|
||||
});
|
||||
|
||||
auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;
|
||||
auto tool_call_body = tool_choice + p.token("</tool_call>") + p.space();
|
||||
p.rule("tool-call", p.token("<tool_call>") + p.literal("\n") + tool_call_body);
|
||||
|
||||
auto tool_call_body = tool_choice + "</tool_call>" + p.space();
|
||||
auto tool_call = p.rule("tool-call", "<tool_call>\n" + tool_call_body);
|
||||
|
||||
// Qwen3-Coder models may occasionally omit the <tool_call> token.
|
||||
auto tool_call_first = is_qwen3_coder ?
|
||||
p.rule("tool-call-first", p.optional(p.literal("<tool_call>\n")) + tool_call_body) :
|
||||
tool_call;
|
||||
|
||||
auto calls = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first;
|
||||
auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1));
|
||||
tool_calls |= p.trigger_rule("tool-calls", p.token("<tool_call>"), p.literal("\n") + tool_call_body + more);
|
||||
|
||||
// The templates separate content from the tool calls with one or two newlines,
|
||||
// stop the content before them
|
||||
std::vector<std::string> content_ends;
|
||||
for (const auto & start : tool_call_starts) {
|
||||
content_ends.insert(content_ends.end(), { "\n\n" + start, "\n" + start, start });
|
||||
}
|
||||
auto content_end = (p.literal("\n\n") + starts) | (p.literal("\n") + starts) | starts;
|
||||
|
||||
return generation_prompt +
|
||||
(reasoning << p.content(p.until_one_of(content_ends)) << tool_calls);
|
||||
return generation_prompt + (reasoning << p.content(p.until(content_end)) << p.repeat(tool_calls, min_calls, 1));
|
||||
}
|
||||
|
||||
// Content only parser
|
||||
@@ -191,17 +180,11 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !has_response_format && has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
|
||||
// The grammar scans for the triggers itself, so the sampler runs it from the first token
|
||||
data.grammar_lazy = false;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
data.parser.build_grammar(builder, !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO);
|
||||
});
|
||||
|
||||
if (data.grammar_lazy) {
|
||||
for (const auto & start : tool_call_starts) {
|
||||
data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, start });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
|
||||
+544
-116
@@ -208,6 +208,89 @@ void common_peg_ast_arena::visit(const common_peg_parse_result & result, const c
|
||||
}
|
||||
}
|
||||
|
||||
common_peg_tokens::common_peg_tokens(const llama_vocab * vocab) {
|
||||
const llama_token n_tokens = llama_vocab_n_tokens(vocab);
|
||||
for (llama_token id = 0; id < n_tokens; ++id) {
|
||||
const auto attr = llama_vocab_get_attr(vocab, id);
|
||||
if (attr & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED)) {
|
||||
std::string text = llama_vocab_get_text(vocab, id);
|
||||
ids.emplace(text, id);
|
||||
tokens.emplace(id, common_peg_token{ std::move(text), attr });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
llama_token common_peg_tokens::token_id(const std::string & text) const {
|
||||
auto it = ids.find(text);
|
||||
return it != ids.end() ? it->second : LLAMA_TOKEN_NULL;
|
||||
}
|
||||
|
||||
// Insert every delimiter the parser matches into the trie, starting from each node in from. Returns the end nodes.
|
||||
// A choice adds the delimiters of each branch. A sequence joins every combination of its children.
|
||||
static std::set<size_t> insert_delimiter(common_trie & trie, const common_peg_arena & arena, common_peg_parser_id id,
|
||||
std::set<size_t> from = { 0 }) {
|
||||
return std::visit([&](const auto & p) -> std::set<size_t> {
|
||||
using T = std::decay_t<decltype(p)>;
|
||||
if constexpr (std::is_same_v<T, common_peg_tag_parser> || std::is_same_v<T, common_peg_atomic_parser>) {
|
||||
return insert_delimiter(trie, arena, p.child, std::move(from));
|
||||
} else if constexpr (std::is_same_v<T, common_peg_literal_parser>) {
|
||||
std::set<size_t> ends;
|
||||
for (size_t node : from) {
|
||||
size_t pos = 0;
|
||||
while (pos < p.literal.size()) {
|
||||
auto result = common_parse_utf8_codepoint(p.literal, pos);
|
||||
if (result.status != utf8_parse_result::SUCCESS) {
|
||||
break;
|
||||
}
|
||||
node = trie.extend(node, common_trie::symbol::codepoint(result.codepoint));
|
||||
pos += result.bytes_consumed;
|
||||
}
|
||||
ends.insert(node);
|
||||
}
|
||||
return ends;
|
||||
} else if constexpr (std::is_same_v<T, common_peg_token_parser>) {
|
||||
std::set<size_t> ends;
|
||||
for (size_t node : from) {
|
||||
ends.insert(trie.extend(node, common_trie::symbol::token(p.token)));
|
||||
}
|
||||
return ends;
|
||||
} else if constexpr (std::is_same_v<T, common_peg_choice_parser>) {
|
||||
std::set<size_t> ends;
|
||||
for (auto child : p.children) {
|
||||
ends.merge(insert_delimiter(trie, arena, child, from));
|
||||
}
|
||||
return ends;
|
||||
} else if constexpr (std::is_same_v<T, common_peg_sequence_parser>) {
|
||||
for (auto child : p.children) {
|
||||
from = insert_delimiter(trie, arena, child, std::move(from));
|
||||
}
|
||||
return from;
|
||||
} else {
|
||||
throw std::invalid_argument("a delimiter may only be a sequence of literals, tokens, and choices of them, possibly wrapped in tags and atomics");
|
||||
}
|
||||
}, arena.get(id));
|
||||
}
|
||||
|
||||
// Build the matcher of an until parser, each pattern valued by the branch it belongs to
|
||||
static common_trie build_until_matcher(const common_peg_arena & arena,
|
||||
const std::vector<common_peg_until_branch> & branches,
|
||||
bool include) {
|
||||
common_trie matcher;
|
||||
for (size_t b = 0; b < branches.size(); b++) {
|
||||
for (size_t end : insert_delimiter(matcher, arena, branches[b].delimiter)) {
|
||||
if (end == 0) {
|
||||
throw std::invalid_argument("delimiter must not match the empty string");
|
||||
}
|
||||
// nothing follows a delimiter without include, so the first branch to claim it wins
|
||||
auto owner = matcher.mark(end, (int32_t) b);
|
||||
if (owner != (int32_t) b && include) {
|
||||
throw std::invalid_argument("until branches " + std::to_string(owner) + " and " + std::to_string(b) + " must not share a delimiter");
|
||||
}
|
||||
}
|
||||
}
|
||||
return matcher;
|
||||
}
|
||||
|
||||
struct parser_executor;
|
||||
|
||||
common_peg_parser_id common_peg_arena::add_parser(common_peg_parser_variant parser) {
|
||||
@@ -298,6 +381,49 @@ struct parser_executor {
|
||||
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos);
|
||||
}
|
||||
|
||||
common_peg_parse_result operator()(const common_peg_token_parser & p) {
|
||||
if (start_pos >= ctx.input.size()) {
|
||||
if (!ctx.is_lenient()) {
|
||||
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
|
||||
}
|
||||
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos);
|
||||
}
|
||||
|
||||
if (start_pos >= ctx.tokens.size() || ctx.tokens[start_pos] != p.token) {
|
||||
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
|
||||
}
|
||||
|
||||
// Skip the rest of the piece, up to the next token or the end of the input
|
||||
auto pos = start_pos + 1;
|
||||
while (pos < ctx.input.size() && ctx.tokens[pos] == LLAMA_TOKEN_NULL) {
|
||||
++pos;
|
||||
}
|
||||
|
||||
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos);
|
||||
}
|
||||
|
||||
common_peg_parse_result operator()(const common_peg_token_not_parser & p) {
|
||||
if (start_pos >= ctx.input.size()) {
|
||||
if (!ctx.is_lenient()) {
|
||||
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
|
||||
}
|
||||
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos);
|
||||
}
|
||||
|
||||
if (start_pos >= ctx.tokens.size() || ctx.tokens[start_pos] == LLAMA_TOKEN_NULL ||
|
||||
std::find(p.tokens.begin(), p.tokens.end(), ctx.tokens[start_pos]) != p.tokens.end()) {
|
||||
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
|
||||
}
|
||||
|
||||
// Skip the rest of the piece, up to the next token or the end of the input
|
||||
auto pos = start_pos + 1;
|
||||
while (pos < ctx.input.size() && ctx.tokens[pos] == LLAMA_TOKEN_NULL) {
|
||||
++pos;
|
||||
}
|
||||
|
||||
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos);
|
||||
}
|
||||
|
||||
common_peg_parse_result operator()(const common_peg_sequence_parser & p) {
|
||||
if (ctx.is_debug()) {
|
||||
LOG_DBG("%sSEQ start at %zu '%s' (%zu children)\n", debug_indent().c_str(), start_pos,
|
||||
@@ -689,7 +815,11 @@ struct parser_executor {
|
||||
}
|
||||
|
||||
common_peg_parse_result operator()(const common_peg_until_parser & p) const {
|
||||
common_trie matcher(p.delimiters);
|
||||
if (p.include) {
|
||||
return parse_through(p);
|
||||
}
|
||||
|
||||
const auto & matcher = p.matcher;
|
||||
|
||||
// Scan input and check for delimiters
|
||||
size_t pos = start_pos;
|
||||
@@ -697,24 +827,24 @@ struct parser_executor {
|
||||
std::vector<common_peg_invalid_utf8> invalid_utf8;
|
||||
|
||||
while (pos < ctx.input.size()) {
|
||||
auto utf8_result = common_parse_utf8_codepoint(ctx.input, pos);
|
||||
auto step = matcher.next(ctx.input, ctx.tokens, pos);
|
||||
|
||||
if (utf8_result.status == utf8_parse_result::INCOMPLETE && ctx.is_lenient()) {
|
||||
if (step.status == utf8_parse_result::INCOMPLETE && ctx.is_lenient()) {
|
||||
// The rest of the sequence may still arrive, return what we have so far
|
||||
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, last_valid_pos, {}, std::move(invalid_utf8));
|
||||
}
|
||||
|
||||
if (utf8_result.status != utf8_parse_result::SUCCESS) {
|
||||
if (step.status != utf8_parse_result::SUCCESS) {
|
||||
// Malformed UTF-8, or a sequence truncated by the end of a complete input.
|
||||
// A delimiter cannot start inside bytes that fail to decode, so consume them and move on
|
||||
invalid_utf8.push_back({pos, utf8_result.bytes_consumed});
|
||||
pos += utf8_result.bytes_consumed;
|
||||
invalid_utf8.push_back({pos, step.bytes_consumed});
|
||||
pos += step.bytes_consumed;
|
||||
last_valid_pos = pos;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if a delimiter starts at this position
|
||||
auto match = matcher.check_at(ctx.input, pos);
|
||||
auto match = matcher.check_at(ctx.input, ctx.tokens, pos);
|
||||
|
||||
if (match == common_trie::COMPLETE_MATCH) {
|
||||
// Found a complete delimiter, return everything before it
|
||||
@@ -727,7 +857,7 @@ struct parser_executor {
|
||||
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, {}, std::move(invalid_utf8));
|
||||
}
|
||||
|
||||
pos += utf8_result.bytes_consumed;
|
||||
pos += step.bytes_consumed;
|
||||
last_valid_pos = pos;
|
||||
}
|
||||
|
||||
@@ -738,6 +868,48 @@ struct parser_executor {
|
||||
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, last_valid_pos, {}, std::move(invalid_utf8));
|
||||
}
|
||||
|
||||
// Parse content, then the first branch whose delimiter and rest follow it, like C (D1 R1 | D2 R2 | ...)
|
||||
common_peg_parse_result parse_through(const common_peg_until_parser & p) const {
|
||||
auto content = arena.parse(p.content, ctx, start_pos);
|
||||
if (content.fail() || content.need_more_input()) {
|
||||
return content;
|
||||
}
|
||||
|
||||
auto pos = content.end;
|
||||
for (const auto & branch : p.branches) {
|
||||
auto nodes = content.nodes;
|
||||
auto invalid_utf8 = content.invalid_utf8;
|
||||
auto end = pos;
|
||||
bool matched = true;
|
||||
|
||||
for (auto id : { branch.delimiter, branch.rest }) {
|
||||
if (id == COMMON_PEG_INVALID_PARSER_ID) {
|
||||
continue;
|
||||
}
|
||||
auto result = arena.parse(id, ctx, end);
|
||||
if (result.fail()) {
|
||||
matched = false;
|
||||
break;
|
||||
}
|
||||
nodes.insert(nodes.end(), result.nodes.begin(), result.nodes.end());
|
||||
invalid_utf8.insert(invalid_utf8.end(), result.invalid_utf8.begin(), result.invalid_utf8.end());
|
||||
end = result.end;
|
||||
if (result.need_more_input()) {
|
||||
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, end, std::move(nodes), std::move(invalid_utf8));
|
||||
}
|
||||
}
|
||||
if (matched) {
|
||||
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, end, std::move(nodes), std::move(invalid_utf8));
|
||||
}
|
||||
}
|
||||
|
||||
if (p.optional && pos == ctx.input.size()) {
|
||||
auto type = ctx.is_lenient() ? COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT : COMMON_PEG_PARSE_RESULT_SUCCESS;
|
||||
return common_peg_parse_result(type, start_pos, pos, std::move(content.nodes), std::move(content.invalid_utf8));
|
||||
}
|
||||
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos, pos);
|
||||
}
|
||||
|
||||
common_peg_parse_result operator()(const common_peg_schema_parser & p) {
|
||||
return arena.parse(p.child, ctx, start_pos);
|
||||
}
|
||||
@@ -823,10 +995,6 @@ struct parser_executor {
|
||||
common_peg_parse_result operator()(const common_peg_gbnf_parser & p) {
|
||||
return arena.parse(p.child, ctx, start_pos);
|
||||
}
|
||||
|
||||
common_peg_parse_result operator()(const common_peg_ac_parser & p) {
|
||||
return arena.parse(p.child, ctx, start_pos);
|
||||
}
|
||||
};
|
||||
|
||||
common_peg_parse_result common_peg_arena::parse(common_peg_parse_context & ctx, size_t start) const {
|
||||
@@ -899,19 +1067,33 @@ void common_peg_arena::resolve_refs() {
|
||||
std::is_same_v<T, common_peg_not_parser> ||
|
||||
std::is_same_v<T, common_peg_tag_parser> ||
|
||||
std::is_same_v<T, common_peg_atomic_parser> ||
|
||||
std::is_same_v<T, common_peg_gbnf_parser> ||
|
||||
std::is_same_v<T, common_peg_ac_parser>) {
|
||||
std::is_same_v<T, common_peg_gbnf_parser>) {
|
||||
p.child = resolve_ref(p.child);
|
||||
} else if constexpr (std::is_same_v<T, common_peg_until_parser>) {
|
||||
if (p.content != COMMON_PEG_INVALID_PARSER_ID) {
|
||||
p.content = resolve_ref(p.content);
|
||||
}
|
||||
for (auto & branch : p.branches) {
|
||||
branch.delimiter = resolve_ref(branch.delimiter);
|
||||
if (branch.rest != COMMON_PEG_INVALID_PARSER_ID) {
|
||||
branch.rest = resolve_ref(branch.rest);
|
||||
}
|
||||
}
|
||||
} else if constexpr (std::is_same_v<T, common_peg_rule_parser>) {
|
||||
p.child = resolve_ref(p.child);
|
||||
if (p.start != COMMON_PEG_INVALID_PARSER_ID) {
|
||||
p.start = resolve_ref(p.start);
|
||||
p.rest = resolve_ref(p.rest);
|
||||
}
|
||||
} else if constexpr (std::is_same_v<T, common_peg_schema_parser>) {
|
||||
p.child = resolve_ref(p.child);
|
||||
} else if constexpr (std::is_same_v<T, common_peg_epsilon_parser> ||
|
||||
std::is_same_v<T, common_peg_start_parser> ||
|
||||
std::is_same_v<T, common_peg_end_parser> ||
|
||||
std::is_same_v<T, common_peg_ref_parser> ||
|
||||
std::is_same_v<T, common_peg_until_parser> ||
|
||||
std::is_same_v<T, common_peg_literal_parser> ||
|
||||
std::is_same_v<T, common_peg_token_parser> ||
|
||||
std::is_same_v<T, common_peg_token_not_parser> ||
|
||||
std::is_same_v<T, common_peg_string_parser> ||
|
||||
std::is_same_v<T, common_peg_chars_parser> ||
|
||||
std::is_same_v<T, common_peg_any_parser> ||
|
||||
@@ -955,6 +1137,14 @@ std::string common_peg_arena::dump_impl(common_peg_parser_id
|
||||
return "End";
|
||||
} else if constexpr (std::is_same_v<T, common_peg_literal_parser>) {
|
||||
return "Literal(" + p.literal + ")";
|
||||
} else if constexpr (std::is_same_v<T, common_peg_token_parser>) {
|
||||
return "Token(" + p.piece + ", " + std::to_string(p.token) + ")";
|
||||
} else if constexpr (std::is_same_v<T, common_peg_token_not_parser>) {
|
||||
std::vector<std::string> parts;
|
||||
for (auto token : p.tokens) {
|
||||
parts.push_back(std::to_string(token));
|
||||
}
|
||||
return "TokenNot(" + string_join(parts, ", ") + ")";
|
||||
} else if constexpr (std::is_same_v<T, common_peg_sequence_parser>) {
|
||||
std::vector<std::string> parts;
|
||||
for (const auto & child : p.children) {
|
||||
@@ -981,8 +1171,6 @@ std::string common_peg_arena::dump_impl(common_peg_parser_id
|
||||
return "Atomic(" + dump_impl(p.child, visited) + ")";
|
||||
} else if constexpr (std::is_same_v<T, common_peg_gbnf_parser>) {
|
||||
return "Gbnf(" + p.grammar + ", " + dump_impl(p.child, visited) + ")";
|
||||
} else if constexpr (std::is_same_v<T, common_peg_ac_parser>) {
|
||||
return "Ac(" + string_join(p.delimiters, " | ") + ", " + dump_impl(p.child, visited) + ")";
|
||||
} else if constexpr (std::is_same_v<T, common_peg_any_parser>) {
|
||||
return "Any";
|
||||
} else if constexpr (std::is_same_v<T, common_peg_space_parser>) {
|
||||
@@ -995,7 +1183,18 @@ std::string common_peg_arena::dump_impl(common_peg_parser_id
|
||||
} else if constexpr (std::is_same_v<T, common_peg_string_parser>) {
|
||||
return "String(" + std::string(1, p.delimiter) + ")";
|
||||
} else if constexpr (std::is_same_v<T, common_peg_until_parser>) {
|
||||
return "Until(" + string_join(p.delimiters, " | ") + ")";
|
||||
std::vector<std::string> branches;
|
||||
for (const auto & branch : p.branches) {
|
||||
auto text = dump_impl(branch.delimiter, visited);
|
||||
if (branch.rest != COMMON_PEG_INVALID_PARSER_ID) {
|
||||
text += " " + dump_impl(branch.rest, visited);
|
||||
}
|
||||
branches.push_back(std::move(text));
|
||||
}
|
||||
if (p.include) {
|
||||
return "Until(" + dump_impl(p.content, visited) + ", " + string_join(branches, " | ") + (p.optional ? ", optional" : "") + ")";
|
||||
}
|
||||
return "Until(" + string_join(branches, " | ") + ")";
|
||||
} else if constexpr (std::is_same_v<T, common_peg_schema_parser>) {
|
||||
return "Schema(" + dump_impl(p.child, visited) + ", " + (p.node ? common_chat_schema::kind_name(p.node->kind()) : "null") + ")";
|
||||
} else if constexpr (std::is_same_v<T, common_peg_rule_parser>) {
|
||||
@@ -1089,6 +1288,67 @@ common_peg_parser operator|(const std::string & str, const common_peg_parser & p
|
||||
|
||||
common_peg_parser_builder::common_peg_parser_builder() {}
|
||||
|
||||
common_peg_parser_builder::common_peg_parser_builder(common_peg_tokens tokens) {
|
||||
arena_.tokens_ = std::move(tokens);
|
||||
}
|
||||
|
||||
common_peg_parser common_peg_parser_builder::until_one_of(const std::vector<std::string> & delimiters) {
|
||||
common_peg_until_parser p;
|
||||
for (const auto & d : delimiters) {
|
||||
p.branches.push_back({ literal(d).id() });
|
||||
}
|
||||
p.matcher = build_until_matcher(arena_, p.branches, p.include);
|
||||
return add(std::move(p));
|
||||
}
|
||||
|
||||
common_peg_parser common_peg_parser_builder::until(const common_peg_parser & delimiter) {
|
||||
common_peg_until_parser p;
|
||||
p.branches = { { delimiter.id() } };
|
||||
p.matcher = build_until_matcher(arena_, p.branches, p.include);
|
||||
return add(std::move(p));
|
||||
}
|
||||
|
||||
common_peg_parser common_peg_parser_builder::through(const common_peg_parser & content, const std::vector<common_peg_until_branch> & branches, bool optional) {
|
||||
if (branches.empty()) {
|
||||
throw std::runtime_error("through requires at least one branch");
|
||||
}
|
||||
common_peg_until_parser p;
|
||||
p.branches = branches;
|
||||
p.include = true;
|
||||
p.optional = optional;
|
||||
p.content = content.id();
|
||||
p.matcher = build_until_matcher(arena_, p.branches, p.include);
|
||||
return add(std::move(p));
|
||||
}
|
||||
|
||||
common_peg_parser common_peg_parser_builder::token(const std::string & piece) {
|
||||
auto token = arena_.tokens_.token_id(piece);
|
||||
if (token == LLAMA_TOKEN_NULL) {
|
||||
return literal(piece);
|
||||
}
|
||||
return add(common_peg_token_parser{token, piece});
|
||||
}
|
||||
|
||||
common_peg_parser common_peg_parser_builder::token(const std::vector<std::string> & pieces, bool negate) {
|
||||
if (!negate) {
|
||||
std::vector<common_peg_parser> alts;
|
||||
for (const auto & piece : pieces) {
|
||||
alts.push_back(token(piece));
|
||||
}
|
||||
return choice(alts);
|
||||
}
|
||||
|
||||
common_peg_token_not_parser p;
|
||||
for (const auto & piece : pieces) {
|
||||
auto token = arena_.tokens_.token_id(piece);
|
||||
if (token == LLAMA_TOKEN_NULL) {
|
||||
throw std::invalid_argument("token is not registered with the builder: " + piece);
|
||||
}
|
||||
p.tokens.push_back(token);
|
||||
}
|
||||
return add(p);
|
||||
}
|
||||
|
||||
common_peg_parser common_peg_parser_builder::sequence(const std::vector<common_peg_parser_id> & parsers) {
|
||||
// Flatten nested sequences
|
||||
std::vector<common_peg_parser_id> flattened;
|
||||
@@ -1195,6 +1455,17 @@ common_peg_parser common_peg_parser_builder::rule(const std::string & name, cons
|
||||
return ref(clean_name);
|
||||
}
|
||||
|
||||
common_peg_parser common_peg_parser_builder::trigger_rule(const std::string & name, const common_peg_parser & start, const common_peg_parser & rest) {
|
||||
common_trie scratch;
|
||||
if (insert_delimiter(scratch, arena_, start.id()).count(0)) {
|
||||
throw std::invalid_argument("trigger start must not match the empty string");
|
||||
}
|
||||
auto clean_name = gbnf_format_rule_name(name);
|
||||
auto rule_id = arena_.add_parser(common_peg_rule_parser{clean_name, (start + rest).id(), true, start.id(), rest.id()});
|
||||
arena_.add_rule(clean_name, rule_id);
|
||||
return ref(clean_name);
|
||||
}
|
||||
|
||||
void common_peg_parser_builder::set_root(const common_peg_parser & p) {
|
||||
arena_.set_root(p.id());
|
||||
}
|
||||
@@ -1392,13 +1663,6 @@ common_peg_parser common_peg_parser_builder::json_member(const std::string & key
|
||||
});
|
||||
}
|
||||
|
||||
common_peg_parser common_peg_parser_builder::ac(const common_peg_parser & p, const std::vector<std::string> & delimiters) {
|
||||
if (delimiters.empty()) {
|
||||
throw std::runtime_error("ac parser requires at least one delimiter");
|
||||
}
|
||||
return add(common_peg_ac_parser{p, delimiters});
|
||||
}
|
||||
|
||||
static std::string gbnf_escape_char_class(uint32_t c) {
|
||||
if (c == '-' || c == ']' || c == '[' || c == '\\') {
|
||||
return "\\" + std::string(1, (char) c);
|
||||
@@ -1457,15 +1721,63 @@ static std::string gbnf_char_class(const std::vector<uint32_t> & chars, bool neg
|
||||
return s + "]";
|
||||
}
|
||||
|
||||
// Token set such as <[1,5-9]>, with consecutive ids collapsed into ranges
|
||||
static std::string gbnf_token_set(const std::vector<uint32_t> & tokens, bool negate) {
|
||||
std::string s = negate ? "!<[" : "<[";
|
||||
for (size_t i = 0; i < tokens.size();) {
|
||||
size_t j = i;
|
||||
while (j + 1 < tokens.size() && tokens[j + 1] == tokens[j] + 1) {
|
||||
j++;
|
||||
}
|
||||
s += (i == 0 ? "" : ",") + std::to_string(tokens[i]);
|
||||
if (j > i) {
|
||||
s += "-" + std::to_string(tokens[j]);
|
||||
}
|
||||
i = j + 1;
|
||||
}
|
||||
return s + "]>";
|
||||
}
|
||||
|
||||
// Symbols on the transitions out of an automaton state, codepoints and tokens kept apart
|
||||
struct gbnf_ac_symbols {
|
||||
std::vector<uint32_t> chars;
|
||||
std::vector<uint32_t> tokens;
|
||||
|
||||
void add(const common_trie::symbol & sym) {
|
||||
(sym.is_token() ? tokens : chars).push_back(sym.value);
|
||||
}
|
||||
|
||||
// One alternative for the codepoints and one for the tokens, each followed by next if given
|
||||
std::vector<std::string> alternatives(const std::string & next = "") const {
|
||||
std::vector<std::string> alts;
|
||||
const std::string tail = next.empty() ? "" : " " + next;
|
||||
if (!chars.empty()) {
|
||||
alts.push_back(gbnf_char_class(chars, false) + tail);
|
||||
}
|
||||
if (!tokens.empty()) {
|
||||
alts.push_back(gbnf_token_set(tokens, false) + tail);
|
||||
}
|
||||
return alts;
|
||||
}
|
||||
};
|
||||
|
||||
static std::string gbnf_ac_grammar(
|
||||
const common_grammar_builder & builder,
|
||||
const std::string & prefix,
|
||||
const std::vector<std::string> & strings,
|
||||
const std::function<std::string(const std::vector<uint32_t> &,
|
||||
const std::map<size_t, std::vector<uint32_t>> &,
|
||||
const std::vector<uint32_t> &,
|
||||
const common_grammar_builder & builder,
|
||||
const std::string & prefix,
|
||||
const common_trie & trie,
|
||||
const std::set<uint32_t> & opaque,
|
||||
const std::function<std::string(const gbnf_ac_symbols &,
|
||||
const std::map<size_t, gbnf_ac_symbols> &,
|
||||
const std::map<size_t, gbnf_ac_symbols> &,
|
||||
const std::string &,
|
||||
const std::function<std::string(size_t)> &)> & build_rule) {
|
||||
common_aho_corasick ac(strings);
|
||||
common_aho_corasick ac(trie);
|
||||
|
||||
// With only tokens in the alphabet, the grammar can step a whole token at a time
|
||||
bool token_only = true;
|
||||
for (const auto & sym : ac.alphabet) {
|
||||
token_only = token_only && sym.is_token();
|
||||
}
|
||||
|
||||
auto state_name = [&](size_t s) -> std::string {
|
||||
if (s == 0) {
|
||||
@@ -1481,27 +1793,43 @@ static std::string gbnf_ac_grammar(
|
||||
continue; // match states
|
||||
}
|
||||
|
||||
std::map<size_t, std::vector<uint32_t>> buckets;
|
||||
std::vector<uint32_t> completing; // chars that complete a delimiter
|
||||
std::vector<uint32_t> specific; // chars with an explicit transition
|
||||
for (uint32_t c : ac.alphabet) {
|
||||
size_t d = ac.next(q, c);
|
||||
std::map<size_t, gbnf_ac_symbols> buckets;
|
||||
std::map<size_t, gbnf_ac_symbols> completing_to; // completing symbols by the match state they reach
|
||||
gbnf_ac_symbols completing; // symbols that complete a delimiter
|
||||
gbnf_ac_symbols specific; // symbols with an explicit transition
|
||||
for (const auto & sym : ac.alphabet) {
|
||||
size_t d = ac.next(q, sym);
|
||||
if (ac.is_terminal(d)) {
|
||||
completing.push_back(c);
|
||||
specific.push_back(c);
|
||||
completing.add(sym);
|
||||
completing_to[d].add(sym);
|
||||
specific.add(sym);
|
||||
} else if (d != 0) {
|
||||
buckets[d].push_back(c); // specific non-root destination
|
||||
specific.push_back(c);
|
||||
buckets[d].add(sym); // specific non-root destination
|
||||
specific.add(sym);
|
||||
}
|
||||
}
|
||||
|
||||
builder.add_rule(state_name(q), build_rule(completing, buckets, specific, state_name));
|
||||
}
|
||||
// every other symbol goes back to the start
|
||||
std::string fallback;
|
||||
if (token_only) {
|
||||
fallback = gbnf_token_set(specific.tokens, true);
|
||||
} else {
|
||||
fallback = specific.chars.empty() ? "." : gbnf_char_class(specific.chars, true);
|
||||
|
||||
// An empty delimiter makes the start state terminal. Emit an entry rule
|
||||
// that matches the empty string so the returned reference stays valid.
|
||||
if (ac.is_terminal(0)) {
|
||||
builder.add_rule(prefix, "|");
|
||||
// Char rules never match a token named by a token rule, so allow the other named tokens by name
|
||||
std::vector<uint32_t> others;
|
||||
for (auto token : opaque) {
|
||||
if (std::find(specific.tokens.begin(), specific.tokens.end(), token) == specific.tokens.end()) {
|
||||
others.push_back(token);
|
||||
}
|
||||
}
|
||||
if (!others.empty()) {
|
||||
fallback = "(" + fallback + " | " + gbnf_token_set(others, false) + ")";
|
||||
}
|
||||
}
|
||||
fallback += " " + state_name(0);
|
||||
|
||||
builder.add_rule(state_name(q), build_rule(completing, completing_to, buckets, fallback, state_name));
|
||||
}
|
||||
|
||||
return state_name(0);
|
||||
@@ -1514,52 +1842,104 @@ static std::string gbnf_ac_grammar(
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/24839
|
||||
static std::string gbnf_excluding_grammar(const common_grammar_builder & builder,
|
||||
const std::string & prefix,
|
||||
const std::vector<std::string> & strings) {
|
||||
return gbnf_ac_grammar(builder, prefix, strings,
|
||||
[](const std::vector<uint32_t> & /*completing*/,
|
||||
const std::map<size_t, std::vector<uint32_t>> & buckets,
|
||||
const std::vector<uint32_t> & specific,
|
||||
const common_trie & trie,
|
||||
const std::set<uint32_t> & opaque) {
|
||||
return gbnf_ac_grammar(builder, prefix, trie, opaque,
|
||||
[](const gbnf_ac_symbols & /*completing*/,
|
||||
const std::map<size_t, gbnf_ac_symbols> & /*completing_to*/,
|
||||
const std::map<size_t, gbnf_ac_symbols> & buckets,
|
||||
const std::string & fallback,
|
||||
const std::function<std::string(size_t)> & state_name) {
|
||||
// every state is accepting and completing chars get no
|
||||
// alternative, so a forbidden string can never be matched
|
||||
// every state accepts and completing symbols get no alternative, so a forbidden string never matches
|
||||
std::string rhs = "|";
|
||||
for (const auto & [d, chars] : buckets) {
|
||||
rhs += " " + gbnf_char_class(chars, false) + " " + state_name(d) + " |";
|
||||
for (const auto & [d, symbols] : buckets) {
|
||||
for (const auto & alt : symbols.alternatives(state_name(d))) {
|
||||
rhs += " " + alt + " |";
|
||||
}
|
||||
}
|
||||
rhs += " " + gbnf_char_class(specific, true) + " " + state_name(0);
|
||||
rhs += " " + fallback;
|
||||
return rhs;
|
||||
});
|
||||
}
|
||||
|
||||
// GBNF grammar matching everything up to and including the first occurrence of
|
||||
// any string in `strings`. Emits the Aho-Corasick automaton DFA and returns
|
||||
// the start state rule name.
|
||||
static std::string gbnf_including_grammar(const common_grammar_builder & builder,
|
||||
const std::string & prefix,
|
||||
const std::vector<std::string> & strings) {
|
||||
return gbnf_ac_grammar(builder, prefix, strings,
|
||||
[](const std::vector<uint32_t> & completing,
|
||||
const std::map<size_t, std::vector<uint32_t>> & buckets,
|
||||
const std::vector<uint32_t> & specific,
|
||||
const std::function<std::string(size_t)> & state_name) {
|
||||
// GBNF grammar matching up to the first pattern in the trie, then the rest rule of that pattern's value.
|
||||
// For the pattern "ab" with rest "x":
|
||||
//
|
||||
// prefix ::= [a] prefix-01 | [^a] prefix
|
||||
// prefix-01 ::= [b] rest | [a] prefix-01 | [^ab] prefix
|
||||
// rest ::= "x"
|
||||
static std::string gbnf_including_grammar(const common_grammar_builder & builder,
|
||||
const std::string & prefix,
|
||||
const common_trie & trie,
|
||||
const std::set<uint32_t> & opaque,
|
||||
const std::vector<std::string> & rests,
|
||||
bool optional) {
|
||||
common_aho_corasick ac(trie);
|
||||
|
||||
// every pattern ending at a match state, the longest and the ones reached by suffix links
|
||||
auto rests_at = [&](size_t s, bool & terminal) {
|
||||
std::vector<std::string> alts;
|
||||
terminal = false;
|
||||
for (; s != 0; s = ac.fail[s]) {
|
||||
auto pattern = ac.t.nodes[s].pattern;
|
||||
if (pattern < 0) {
|
||||
continue;
|
||||
}
|
||||
if (rests[pattern].empty()) {
|
||||
terminal = true;
|
||||
} else if (std::find(alts.begin(), alts.end(), rests[pattern]) == alts.end()) {
|
||||
alts.push_back(rests[pattern]);
|
||||
}
|
||||
}
|
||||
return alts;
|
||||
};
|
||||
|
||||
return gbnf_ac_grammar(builder, prefix, trie, opaque,
|
||||
[&](const gbnf_ac_symbols & /*completing*/,
|
||||
const std::map<size_t, gbnf_ac_symbols> & completing_to,
|
||||
const std::map<size_t, gbnf_ac_symbols> & buckets,
|
||||
const std::string & fallback,
|
||||
const std::function<std::string(size_t)> & state_name) {
|
||||
std::vector<std::string> alts;
|
||||
if (!completing.empty()) {
|
||||
alts.push_back(gbnf_char_class(completing, false)); // terminate on match
|
||||
gbnf_ac_symbols terminating; // symbols that complete a pattern with no rest
|
||||
for (const auto & [d, symbols] : completing_to) {
|
||||
bool terminal = false;
|
||||
auto rest = rests_at(d, terminal);
|
||||
if (!rest.empty()) {
|
||||
for (auto & alt : symbols.alternatives(rest.size() == 1 ? rest[0] : "(" + string_join(rest, " | ") + ")")) {
|
||||
alts.push_back(std::move(alt));
|
||||
}
|
||||
}
|
||||
if (terminal) {
|
||||
terminating.chars.insert(terminating.chars.end(), symbols.chars.begin(), symbols.chars.end());
|
||||
terminating.tokens.insert(terminating.tokens.end(), symbols.tokens.begin(), symbols.tokens.end());
|
||||
}
|
||||
}
|
||||
for (const auto & [d, chars] : buckets) {
|
||||
alts.push_back(gbnf_char_class(chars, false) + " " + state_name(d));
|
||||
for (auto & alt : terminating.alternatives()) {
|
||||
alts.push_back(std::move(alt));
|
||||
}
|
||||
// every other character keeps scanning from the start state
|
||||
alts.push_back(gbnf_char_class(specific, true) + " " + state_name(0));
|
||||
return string_join(alts, " | ");
|
||||
for (const auto & [d, symbols] : buckets) {
|
||||
for (auto & alt : symbols.alternatives(state_name(d))) {
|
||||
alts.push_back(std::move(alt));
|
||||
}
|
||||
}
|
||||
// every other symbol keeps scanning from the start state
|
||||
alts.push_back(fallback);
|
||||
return (optional ? "| " : "") + string_join(alts, " | ");
|
||||
});
|
||||
}
|
||||
|
||||
static std::set<std::string> collect_reachable_rules(
|
||||
// The rules and the tokens named by token rules that the grammar of a parser reaches
|
||||
struct common_peg_reachable {
|
||||
std::set<std::string> rules;
|
||||
std::set<uint32_t> tokens;
|
||||
};
|
||||
|
||||
static void collect_reachable(
|
||||
const common_peg_arena & arena,
|
||||
const common_peg_parser_id & rule
|
||||
const common_peg_parser_id & root,
|
||||
common_peg_reachable & reachable
|
||||
) {
|
||||
std::set<std::string> reachable;
|
||||
std::set<std::string> visited;
|
||||
|
||||
std::function<void(common_peg_parser_id)> visit = [&](common_peg_parser_id id) {
|
||||
@@ -1571,13 +1951,27 @@ static std::set<std::string> collect_reachable_rules(
|
||||
if constexpr (std::is_same_v<T, common_peg_epsilon_parser> ||
|
||||
std::is_same_v<T, common_peg_start_parser> ||
|
||||
std::is_same_v<T, common_peg_end_parser> ||
|
||||
std::is_same_v<T, common_peg_until_parser> ||
|
||||
std::is_same_v<T, common_peg_literal_parser> ||
|
||||
std::is_same_v<T, common_peg_chars_parser> ||
|
||||
std::is_same_v<T, common_peg_space_parser> ||
|
||||
std::is_same_v<T, common_peg_any_parser> ||
|
||||
std::is_same_v<T, common_peg_string_parser>) {
|
||||
// These parsers do not have any children
|
||||
} else if constexpr (std::is_same_v<T, common_peg_token_parser>) {
|
||||
reachable.tokens.insert(p.token);
|
||||
} else if constexpr (std::is_same_v<T, common_peg_token_not_parser>) {
|
||||
reachable.tokens.insert(p.tokens.begin(), p.tokens.end());
|
||||
} else if constexpr (std::is_same_v<T, common_peg_until_parser>) {
|
||||
// The delimiters name their tokens in the grammar
|
||||
if (p.content != COMMON_PEG_INVALID_PARSER_ID) {
|
||||
visit(p.content);
|
||||
}
|
||||
for (const auto & branch : p.branches) {
|
||||
visit(branch.delimiter);
|
||||
if (branch.rest != COMMON_PEG_INVALID_PARSER_ID) {
|
||||
visit(branch.rest);
|
||||
}
|
||||
}
|
||||
} else if constexpr (std::is_same_v<T, common_peg_sequence_parser>) {
|
||||
for (auto child : p.children) {
|
||||
visit(child);
|
||||
@@ -1592,13 +1986,12 @@ static std::set<std::string> collect_reachable_rules(
|
||||
std::is_same_v<T, common_peg_tag_parser> ||
|
||||
std::is_same_v<T, common_peg_atomic_parser> ||
|
||||
std::is_same_v<T, common_peg_gbnf_parser> ||
|
||||
std::is_same_v<T, common_peg_ac_parser> ||
|
||||
std::is_same_v<T, common_peg_schema_parser>) {
|
||||
visit(p.child);
|
||||
} else if constexpr (std::is_same_v<T, common_peg_rule_parser>) {
|
||||
if (visited.find(p.name) == visited.end()) {
|
||||
visited.insert(p.name);
|
||||
reachable.insert(p.name);
|
||||
reachable.rules.insert(p.name);
|
||||
visit(p.child);
|
||||
}
|
||||
} else if constexpr (std::is_same_v<T, common_peg_ref_parser>) {
|
||||
@@ -1611,12 +2004,13 @@ static std::set<std::string> collect_reachable_rules(
|
||||
}, parser);
|
||||
};
|
||||
|
||||
visit(rule);
|
||||
return reachable;
|
||||
visit(root);
|
||||
}
|
||||
|
||||
// GBNF generation implementation
|
||||
void common_peg_arena::build_grammar(const common_grammar_builder & builder, bool lazy) const {
|
||||
common_peg_reachable reachable;
|
||||
|
||||
// A raw string value is parsed by the child rather than constrained by the schema
|
||||
auto schema_delegates = [](const common_peg_schema_parser & s) -> bool {
|
||||
return !s.node || (s.raw && s.node->may_be_string());
|
||||
@@ -1655,6 +2049,10 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo
|
||||
return "";
|
||||
} else if constexpr (std::is_same_v<T, common_peg_literal_parser>) {
|
||||
return gbnf_format_literal(p.literal);
|
||||
} else if constexpr (std::is_same_v<T, common_peg_token_parser>) {
|
||||
return "<[" + std::to_string(p.token) + "]>";
|
||||
} else if constexpr (std::is_same_v<T, common_peg_token_not_parser>) {
|
||||
return gbnf_token_set(std::vector<uint32_t>(p.tokens.begin(), p.tokens.end()), true);
|
||||
} else if constexpr (std::is_same_v<T, common_peg_sequence_parser>) {
|
||||
std::string s;
|
||||
for (const auto & child : p.children) {
|
||||
@@ -1746,10 +2144,19 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo
|
||||
const std::string delim(1, p.delimiter);
|
||||
return R"(( [^)" + delim + R"(\\] | "\\" ( [)" + delim + R"(\\/ bfnrt] | "u" [0-9a-fA-F]{4} ) )*)";
|
||||
} else if constexpr (std::is_same_v<T, common_peg_until_parser>) {
|
||||
if (p.delimiters.empty()) {
|
||||
return ".*";
|
||||
auto prefix = "until-" + std::to_string(id);
|
||||
if (!p.include) {
|
||||
if (p.branches.empty()) {
|
||||
return ".*";
|
||||
}
|
||||
return gbnf_excluding_grammar(builder, prefix, p.matcher, reachable.tokens);
|
||||
}
|
||||
return gbnf_excluding_grammar(builder, "until-" + std::to_string(id), p.delimiters);
|
||||
std::vector<std::string> branch_rests;
|
||||
for (size_t b = 0; b < p.branches.size(); b++) {
|
||||
auto rest = p.branches[b].rest;
|
||||
branch_rests.push_back(rest == COMMON_PEG_INVALID_PARSER_ID ? "" : builder.add_rule(prefix + "-rest-" + std::to_string(b), to_gbnf(rest)));
|
||||
}
|
||||
return gbnf_including_grammar(builder, prefix, p.matcher, reachable.tokens, branch_rests, p.optional);
|
||||
} else if constexpr (std::is_same_v<T, common_peg_schema_parser>) {
|
||||
if (schema_delegates(p)) {
|
||||
return to_gbnf(p.child);
|
||||
@@ -1766,38 +2173,52 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo
|
||||
return to_gbnf(p.child);
|
||||
} else if constexpr (std::is_same_v<T, common_peg_gbnf_parser>) {
|
||||
return p.grammar;
|
||||
} else if constexpr (std::is_same_v<T, common_peg_ac_parser>) {
|
||||
return gbnf_including_grammar(builder, "ac-" + std::to_string(id), p.delimiters);
|
||||
} else {
|
||||
static_assert(is_always_false_v<T>);
|
||||
}
|
||||
}, parser);
|
||||
};
|
||||
|
||||
// Collect reachable rules
|
||||
std::set<std::string> reachable_rules;
|
||||
|
||||
if (lazy) {
|
||||
// Collect rules reachable from trigger rules
|
||||
for (const auto & [name, id] : rules_) {
|
||||
const auto & parser = parsers_.at(id);
|
||||
if (auto rule = std::get_if<common_peg_rule_parser>(&parser)) {
|
||||
if (rule->trigger) {
|
||||
// Mark trigger as reachable and visit it
|
||||
reachable_rules.insert(name);
|
||||
auto add_rules = collect_reachable_rules(*this, id);
|
||||
reachable_rules.insert(add_rules.begin(), add_rules.end());
|
||||
}
|
||||
// Trigger rules, sorted for a predictable order
|
||||
std::vector<const common_peg_rule_parser *> trigger_rules;
|
||||
for (const auto & [name, id] : rules_) {
|
||||
if (auto rule = std::get_if<common_peg_rule_parser>(&parsers_.at(id))) {
|
||||
if (rule->trigger) {
|
||||
trigger_rules.push_back(rule);
|
||||
}
|
||||
}
|
||||
}
|
||||
std::sort(trigger_rules.begin(), trigger_rules.end(), [](const auto * a, const auto * b) { return a->name < b->name; });
|
||||
|
||||
// A lazy grammar scans for the starts when the trigger rules were built from a start and rest
|
||||
bool scans = false;
|
||||
for (const auto * rule : trigger_rules) {
|
||||
scans = scans || rule->start != COMMON_PEG_INVALID_PARSER_ID;
|
||||
}
|
||||
if (scans) {
|
||||
for (const auto * rule : trigger_rules) {
|
||||
if (rule->start == COMMON_PEG_INVALID_PARSER_ID) {
|
||||
throw std::runtime_error("trigger rule without a start in a scanning grammar: " + rule->name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (lazy && scans) {
|
||||
for (const auto * rule : trigger_rules) {
|
||||
collect_reachable(*this, rule->start, reachable);
|
||||
collect_reachable(*this, rule->rest, reachable);
|
||||
}
|
||||
} else if (lazy) {
|
||||
for (const auto * rule : trigger_rules) {
|
||||
collect_reachable(*this, rules_.at(rule->name), reachable);
|
||||
}
|
||||
} else {
|
||||
// Collect rules reachable from root
|
||||
reachable_rules = collect_reachable_rules(*this, root_);
|
||||
collect_reachable(*this, root_, reachable);
|
||||
}
|
||||
|
||||
// Create GBNF rules for all reachable rules
|
||||
for (const auto & [name, rule_id] : rules_) {
|
||||
if (reachable_rules.find(name) == reachable_rules.end()) {
|
||||
if (reachable.rules.find(name) == reachable.rules.end()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1807,20 +2228,27 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo
|
||||
}
|
||||
}
|
||||
|
||||
if (lazy) {
|
||||
// Generate root rule from trigger rules only
|
||||
std::vector<std::string> trigger_names;
|
||||
for (const auto & [name, rule_id] : rules_) {
|
||||
const auto & parser = parsers_.at(rule_id);
|
||||
if (auto rule = std::get_if<common_peg_rule_parser>(&parser)) {
|
||||
if (rule->trigger) {
|
||||
trigger_names.push_back(rule->name);
|
||||
if (lazy && scans) {
|
||||
// The root scans for a trigger start and then constrains its rest
|
||||
common_trie starts;
|
||||
std::vector<std::string> rests;
|
||||
for (const auto * rule : trigger_rules) {
|
||||
auto value = (int32_t) rests.size();
|
||||
rests.push_back(builder.add_rule(rule->name + "-rest", to_gbnf(rule->rest)));
|
||||
for (size_t end : insert_delimiter(starts, *this, rule->start)) {
|
||||
auto owner = starts.mark(end, value);
|
||||
if (owner != value) {
|
||||
throw std::runtime_error("trigger rules " + trigger_rules[owner]->name + " and " + rule->name + " must not share a start");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sort for predictable order
|
||||
std::sort(trigger_names.begin(), trigger_names.end());
|
||||
builder.add_rule("root", gbnf_including_grammar(builder, "trigger", starts, reachable.tokens, rests, /* optional = */ true));
|
||||
} else if (lazy) {
|
||||
// Generate root rule from trigger rules only
|
||||
std::vector<std::string> trigger_names;
|
||||
for (const auto * rule : trigger_rules) {
|
||||
trigger_names.push_back(rule->name);
|
||||
}
|
||||
builder.add_rule("root", string_join(trigger_names, " | "));
|
||||
} else if (root_ != COMMON_PEG_INVALID_PARSER_ID) {
|
||||
builder.add_rule("root", to_gbnf(root_));
|
||||
|
||||
+77
-17
@@ -3,6 +3,7 @@
|
||||
#include "json-schema.h"
|
||||
#include "json.h"
|
||||
#include "llama.h"
|
||||
#include "trie.h"
|
||||
|
||||
#include <memory>
|
||||
#include <set>
|
||||
@@ -188,6 +189,22 @@ struct common_peg_recursion_error : std::invalid_argument {
|
||||
using std::invalid_argument::invalid_argument;
|
||||
};
|
||||
|
||||
struct common_peg_token {
|
||||
std::string text;
|
||||
llama_token_attr attr;
|
||||
};
|
||||
|
||||
struct common_peg_tokens {
|
||||
std::unordered_map<llama_token, common_peg_token> tokens;
|
||||
std::unordered_map<std::string, llama_token> ids;
|
||||
|
||||
common_peg_tokens() = default;
|
||||
explicit common_peg_tokens(const llama_vocab * vocab);
|
||||
|
||||
// LLAMA_TOKEN_NULL if text is not a special token
|
||||
llama_token token_id(const std::string & text) const;
|
||||
};
|
||||
|
||||
struct common_peg_parse_context {
|
||||
std::string input; // [h, e, l, l, o, _, w, o, r, l, d]
|
||||
std::vector<llama_token> tokens; // [id, -1, -1, -1, -1, id, -1, -1, -1, -1, -1]
|
||||
@@ -226,6 +243,16 @@ struct common_peg_literal_parser {
|
||||
std::string literal;
|
||||
};
|
||||
|
||||
struct common_peg_token_parser {
|
||||
llama_token token;
|
||||
std::string piece;
|
||||
};
|
||||
|
||||
// Matches one token that is not one of the tokens
|
||||
struct common_peg_token_not_parser {
|
||||
std::vector<llama_token> tokens;
|
||||
};
|
||||
|
||||
struct common_peg_sequence_parser {
|
||||
std::vector<common_peg_parser_id> children;
|
||||
};
|
||||
@@ -270,8 +297,18 @@ struct common_peg_string_parser {
|
||||
char delimiter;
|
||||
};
|
||||
|
||||
// A delimiter an until parser stops at, and the parser that follows it when the delimiter is consumed
|
||||
struct common_peg_until_branch {
|
||||
common_peg_parser_id delimiter;
|
||||
common_peg_parser_id rest = COMMON_PEG_INVALID_PARSER_ID;
|
||||
};
|
||||
|
||||
struct common_peg_until_parser {
|
||||
std::vector<std::string> delimiters;
|
||||
std::vector<common_peg_until_branch> branches;
|
||||
bool include = false; // consume the first delimiter and parse the rest of its branch
|
||||
bool optional = false; // with include, the input may end before any delimiter
|
||||
common_peg_parser_id content = COMMON_PEG_INVALID_PARSER_ID; // with include, parsed up to the delimiter
|
||||
common_trie matcher; // each pattern valued by its branch
|
||||
};
|
||||
|
||||
struct common_peg_schema_parser {
|
||||
@@ -288,6 +325,9 @@ struct common_peg_rule_parser {
|
||||
std::string name;
|
||||
common_peg_parser_id child;
|
||||
bool trigger;
|
||||
// set when the rule has a start and rest, the lazy grammar scans for the start
|
||||
common_peg_parser_id start = COMMON_PEG_INVALID_PARSER_ID;
|
||||
common_peg_parser_id rest = COMMON_PEG_INVALID_PARSER_ID;
|
||||
};
|
||||
|
||||
struct common_peg_ref_parser {
|
||||
@@ -308,17 +348,14 @@ struct common_peg_gbnf_parser {
|
||||
std::string grammar;
|
||||
};
|
||||
|
||||
struct common_peg_ac_parser {
|
||||
common_peg_parser_id child;
|
||||
std::vector<std::string> delimiters;
|
||||
};
|
||||
|
||||
// Variant holding all parser types
|
||||
using common_peg_parser_variant = std::variant<
|
||||
common_peg_epsilon_parser,
|
||||
common_peg_start_parser,
|
||||
common_peg_end_parser,
|
||||
common_peg_literal_parser,
|
||||
common_peg_token_parser,
|
||||
common_peg_token_not_parser,
|
||||
common_peg_sequence_parser,
|
||||
common_peg_choice_parser,
|
||||
common_peg_repetition_parser,
|
||||
@@ -334,14 +371,14 @@ using common_peg_parser_variant = std::variant<
|
||||
common_peg_ref_parser,
|
||||
common_peg_atomic_parser,
|
||||
common_peg_tag_parser,
|
||||
common_peg_gbnf_parser,
|
||||
common_peg_ac_parser
|
||||
common_peg_gbnf_parser
|
||||
>;
|
||||
|
||||
class common_peg_arena {
|
||||
std::vector<common_peg_parser_variant> parsers_;
|
||||
std::unordered_map<std::string, common_peg_parser_id> rules_;
|
||||
common_peg_parser_id root_ = COMMON_PEG_INVALID_PARSER_ID;
|
||||
common_peg_tokens tokens_;
|
||||
|
||||
public:
|
||||
const common_peg_parser_variant & get(common_peg_parser_id id) const { return parsers_.at(id); }
|
||||
@@ -356,6 +393,8 @@ class common_peg_arena {
|
||||
common_peg_parser_id root() const { return root_; }
|
||||
void set_root(common_peg_parser_id id) { root_ = id; }
|
||||
|
||||
const common_peg_tokens & tokens() const { return tokens_; }
|
||||
|
||||
common_peg_parse_result parse(common_peg_parse_context & ctx, size_t start = 0) const;
|
||||
common_peg_parse_result parse(common_peg_parser_id id, common_peg_parse_context & ctx, size_t start) const;
|
||||
|
||||
@@ -384,6 +423,7 @@ class common_peg_parser_builder {
|
||||
|
||||
public:
|
||||
common_peg_parser_builder();
|
||||
explicit common_peg_parser_builder(common_peg_tokens tokens);
|
||||
|
||||
// Match nothing, always succeed.
|
||||
// S -> ε
|
||||
@@ -401,6 +441,16 @@ class common_peg_parser_builder {
|
||||
// S -> "hello"
|
||||
common_peg_parser literal(const std::string & literal) { return add(common_peg_literal_parser{literal}); }
|
||||
|
||||
// Matches a token. Falls back to a literal if the token is not registered with the builder.
|
||||
// S -> <[token-id]>
|
||||
common_peg_parser token(const std::string & piece);
|
||||
|
||||
// Matches one of the tokens. With negate, matches one token that is not in the set.
|
||||
// A negated set may only name tokens registered with the builder.
|
||||
// S -> <[id1]> | <[id2]> | ... or !<[id1,id2,...]>
|
||||
common_peg_parser token(const std::vector<std::string> & pieces, bool negate = false);
|
||||
common_peg_parser token(std::initializer_list<std::string> pieces, bool negate = false) { return token(std::vector<std::string>(pieces), negate); }
|
||||
|
||||
// Matches a sequence of parsers in order, all must succeed.
|
||||
// S -> A B C
|
||||
common_peg_parser sequence() { return add(common_peg_sequence_parser{}); }
|
||||
@@ -457,16 +507,29 @@ class common_peg_parser_builder {
|
||||
// Matches all characters until a delimiter is found (delimiter not consumed).
|
||||
// Invalid UTF-8 is consumed and recorded on the AST nodes.
|
||||
// S -> (!delim .)*
|
||||
common_peg_parser until(const std::string & delimiter) { return add(common_peg_until_parser{{delimiter}}); }
|
||||
common_peg_parser until(const std::string & delimiter) { return until_one_of({delimiter}); }
|
||||
|
||||
// Matches all characters until one of the delimiters in the list is found (delimiter not consumed).
|
||||
// S -> (!delim .)*
|
||||
common_peg_parser until_one_of(const std::vector<std::string> & delimiters) { return add(common_peg_until_parser{delimiters}); }
|
||||
common_peg_parser until_one_of(const std::vector<std::string> & delimiters);
|
||||
|
||||
// Matches all characters until the delimiter is found (delimiter not consumed).
|
||||
// The delimiter may only be a sequence of literals, tokens, and choices of them. Tags and atomics may wrap them.
|
||||
// S -> (!delim .)*
|
||||
common_peg_parser until(const common_peg_parser & delimiter);
|
||||
|
||||
// Matches everything
|
||||
// S -> .*
|
||||
common_peg_parser rest() { return until_one_of({}); }
|
||||
|
||||
// Matches content, then the first delimiter, then the rest of the branch of that delimiter.
|
||||
// Branches must not share a delimiter. With optional, the input may end before any delimiter.
|
||||
// S -> C (D1 R1 | D2 R2 | ...)
|
||||
common_peg_parser through(const common_peg_parser & content, const std::vector<common_peg_until_branch> & branches, bool optional = false);
|
||||
common_peg_parser through(const common_peg_parser & content, std::initializer_list<common_peg_until_branch> branches, bool optional = false) { return through(content, std::vector<common_peg_until_branch>(branches), optional); }
|
||||
common_peg_parser through(const common_peg_parser & content, const common_peg_parser & delimiter) { return through(content, { { delimiter } }); }
|
||||
common_peg_parser through(const common_peg_parser & content, const std::string & delimiter) { return through(content, literal(delimiter)); }
|
||||
|
||||
// Matches between min and max repetitions of a parser (inclusive).
|
||||
// S -> A{m,n}
|
||||
// Use -1 for max to represent unbounded repetition (equivalent to {m,})
|
||||
@@ -537,6 +600,10 @@ class common_peg_parser_builder {
|
||||
common_peg_parser trigger_rule(const std::string & name, const common_peg_parser & p) { return rule(name, p, true); }
|
||||
common_peg_parser trigger_rule(const std::string & name, const std::function<common_peg_parser()> & builder) { return rule(name, builder, true); }
|
||||
|
||||
// Creates a trigger rule that matches start followed by rest. A lazy grammar scans for the start of any such rule, then only allows its rest.
|
||||
// The start has the same limits as an until() delimiter.
|
||||
common_peg_parser trigger_rule(const std::string & name, const common_peg_parser & start, const common_peg_parser & rest);
|
||||
|
||||
// Creates an atomic parser. Atomic parsers do not create an AST node if
|
||||
// the child results in a partial parse, i.e. NEEDS_MORE_INPUT. This is
|
||||
// intended for situations where partial output is undesirable.
|
||||
@@ -550,13 +617,6 @@ class common_peg_parser_builder {
|
||||
// the child's grammar. Parsing delegates entirely to the child.
|
||||
common_peg_parser gbnf(const common_peg_parser & p, const std::string & grammar) { return add(common_peg_gbnf_parser{p, grammar}); }
|
||||
|
||||
// Wraps a child parser but emits a GBNF grammar built from the Aho-Corasick
|
||||
// automaton of `delimiters`, matching everything up to and including the
|
||||
// first delimiter. Parsing delegates entirely to the child, which is
|
||||
// responsible for consuming the delimiter (e.g. until(D) + literal(D)).
|
||||
common_peg_parser ac(const common_peg_parser & p, const std::vector<std::string> & delimiters);
|
||||
common_peg_parser ac(const common_peg_parser & p, const std::string & delimiter) { return ac(p, std::vector<std::string>{delimiter}); }
|
||||
|
||||
void set_root(const common_peg_parser & p);
|
||||
|
||||
common_peg_arena build();
|
||||
|
||||
@@ -31,14 +31,18 @@ struct token_matcher {
|
||||
static common_trie build_trie(const std::vector<llama_tokens> & seqs) {
|
||||
common_trie t;
|
||||
for (const auto & seq : seqs) {
|
||||
t.insert(std::vector<uint32_t>(seq.begin(), seq.end()));
|
||||
std::vector<common_trie::symbol> symbols;
|
||||
for (llama_token token : seq) {
|
||||
symbols.push_back(common_trie::symbol::token(token));
|
||||
}
|
||||
t.insert(symbols);
|
||||
}
|
||||
return t;
|
||||
}
|
||||
|
||||
// returns the index into seqs of the longest sequence ending at this token, or -1
|
||||
int32_t advance(llama_token token) {
|
||||
state = ac.next(state, (uint32_t) token);
|
||||
state = ac.next(state, common_trie::symbol::token(token));
|
||||
const int32_t p = ac.match_pattern(state);
|
||||
if (p >= 0) {
|
||||
state = 0;
|
||||
|
||||
+54
-20
@@ -4,26 +4,46 @@
|
||||
|
||||
#include <deque>
|
||||
|
||||
common_trie::step common_trie::next(std::string_view sv, const std::vector<int32_t> & token_map, size_t pos) const {
|
||||
if (!tokens.empty() && pos < token_map.size()) {
|
||||
const int32_t id = token_map[pos];
|
||||
if (id >= 0 && tokens.find(id) != tokens.end()) {
|
||||
size_t end = pos + 1;
|
||||
while (end < token_map.size() && token_map[end] < 0) {
|
||||
++end;
|
||||
}
|
||||
return { utf8_parse_result::SUCCESS, symbol::token(id), end - pos };
|
||||
}
|
||||
}
|
||||
|
||||
auto result = common_parse_utf8_codepoint(sv, pos);
|
||||
return { result.status, symbol::codepoint(result.codepoint), result.bytes_consumed };
|
||||
}
|
||||
|
||||
common_trie::match_result common_trie::check_at(std::string_view sv, size_t start_pos) const {
|
||||
return check_at(sv, {}, start_pos);
|
||||
}
|
||||
|
||||
common_trie::match_result common_trie::check_at(std::string_view sv, const std::vector<int32_t> & token_map, size_t start_pos) const {
|
||||
size_t current = 0; // Start at root
|
||||
size_t pos = start_pos;
|
||||
|
||||
// LOG_DBG("%s: checking at pos %zu, sv='%s'\n", __func__, start_pos, std::string(sv).c_str());
|
||||
|
||||
while (pos < sv.size()) {
|
||||
auto result = common_parse_utf8_codepoint(sv, pos);
|
||||
if (result.status != utf8_parse_result::SUCCESS) {
|
||||
auto step = next(sv, token_map, pos);
|
||||
if (step.status != utf8_parse_result::SUCCESS) {
|
||||
break;
|
||||
}
|
||||
|
||||
auto it = nodes[current].children.find(result.codepoint);
|
||||
auto it = nodes[current].children.find(step.symbol);
|
||||
if (it == nodes[current].children.end()) {
|
||||
// Can't continue matching
|
||||
return match_result{match_result::NO_MATCH};
|
||||
}
|
||||
|
||||
current = it->second;
|
||||
pos += result.bytes_consumed;
|
||||
pos += step.bytes_consumed;
|
||||
|
||||
// Check if we've matched a complete word
|
||||
if (nodes[current].pattern >= 0) {
|
||||
@@ -42,7 +62,7 @@ common_trie::match_result common_trie::check_at(std::string_view sv, size_t star
|
||||
}
|
||||
|
||||
int32_t common_trie::insert(const std::string & word) {
|
||||
std::vector<uint32_t> symbols;
|
||||
std::vector<symbol> symbols;
|
||||
size_t pos = 0;
|
||||
while (pos < word.length()) {
|
||||
auto result = common_parse_utf8_codepoint(word, pos);
|
||||
@@ -50,28 +70,42 @@ int32_t common_trie::insert(const std::string & word) {
|
||||
break;
|
||||
}
|
||||
|
||||
symbols.push_back(result.codepoint);
|
||||
symbols.push_back(symbol::codepoint(result.codepoint));
|
||||
pos += result.bytes_consumed;
|
||||
}
|
||||
return insert(symbols);
|
||||
}
|
||||
|
||||
int32_t common_trie::insert(const std::vector<uint32_t> & symbols) {
|
||||
int32_t common_trie::insert(const std::vector<symbol> & symbols) {
|
||||
size_t current = 0;
|
||||
for (uint32_t ch : symbols) {
|
||||
auto it = nodes[current].children.find(ch);
|
||||
if (it == nodes[current].children.end()) {
|
||||
size_t child = create_node();
|
||||
nodes[current].children[ch] = child;
|
||||
current = child;
|
||||
} else {
|
||||
current = it->second;
|
||||
}
|
||||
for (symbol ch : symbols) {
|
||||
current = extend(current, ch);
|
||||
}
|
||||
if (nodes[current].pattern < 0) {
|
||||
nodes[current].pattern = n_patterns++;
|
||||
int32_t pattern = mark(current, n_patterns);
|
||||
if (pattern == n_patterns) {
|
||||
n_patterns++;
|
||||
}
|
||||
return nodes[current].pattern;
|
||||
return pattern;
|
||||
}
|
||||
|
||||
size_t common_trie::extend(size_t node, symbol ch) {
|
||||
if (ch.is_token()) {
|
||||
tokens.insert((int32_t) ch.value);
|
||||
}
|
||||
auto it = nodes[node].children.find(ch);
|
||||
if (it != nodes[node].children.end()) {
|
||||
return it->second;
|
||||
}
|
||||
size_t child = create_node();
|
||||
nodes[node].children[ch] = child;
|
||||
return child;
|
||||
}
|
||||
|
||||
int32_t common_trie::mark(size_t node, int32_t value) {
|
||||
if (nodes[node].pattern < 0) {
|
||||
nodes[node].pattern = value;
|
||||
}
|
||||
return nodes[node].pattern;
|
||||
}
|
||||
|
||||
common_aho_corasick::common_aho_corasick(common_trie trie) : t(std::move(trie)) {
|
||||
@@ -113,7 +147,7 @@ common_aho_corasick::common_aho_corasick(common_trie trie) : t(std::move(trie))
|
||||
}
|
||||
}
|
||||
|
||||
size_t common_aho_corasick::next(size_t state, uint32_t ch) const {
|
||||
size_t common_aho_corasick::next(size_t state, common_trie::symbol ch) const {
|
||||
const auto & nodes = t.nodes;
|
||||
while (state && nodes[state].children.find(ch) == nodes[state].children.end()) {
|
||||
state = fail[state];
|
||||
|
||||
+51
-10
@@ -1,5 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include "unicode.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <set>
|
||||
@@ -10,13 +12,42 @@
|
||||
// Trie for matching multiple literals.
|
||||
// This is used in common_peg_until_parser and to build a GBNF exclusion grammar
|
||||
struct common_trie {
|
||||
// A symbol is either a UTF-8 codepoint or a token
|
||||
struct symbol {
|
||||
enum kind_type : uint8_t { CODEPOINT, TOKEN };
|
||||
|
||||
kind_type kind = CODEPOINT;
|
||||
uint32_t value = 0;
|
||||
|
||||
static symbol codepoint(uint32_t cpt) { return { CODEPOINT, cpt }; }
|
||||
static symbol token(int32_t id) { return { TOKEN, (uint32_t) id }; }
|
||||
|
||||
bool is_token() const { return kind == TOKEN; }
|
||||
|
||||
bool operator<(const symbol & other) const {
|
||||
return kind != other.kind ? kind < other.kind : value < other.value;
|
||||
}
|
||||
bool operator==(const symbol & other) const {
|
||||
return kind == other.kind && value == other.value;
|
||||
}
|
||||
};
|
||||
|
||||
struct step {
|
||||
enum utf8_parse_result::status status;
|
||||
common_trie::symbol symbol;
|
||||
size_t bytes_consumed;
|
||||
};
|
||||
|
||||
struct node {
|
||||
std::map<uint32_t, size_t> children; // Use uint32_t to store Unicode codepoints
|
||||
std::map<symbol, size_t> children;
|
||||
int32_t pattern = -1; // index of the pattern ending at this node, -1 if none
|
||||
};
|
||||
|
||||
std::vector<node> nodes;
|
||||
|
||||
// every token id used by a pattern, the input is split so these match as single symbols
|
||||
std::set<int32_t> tokens;
|
||||
|
||||
common_trie() {
|
||||
create_node(); // root node
|
||||
}
|
||||
@@ -29,15 +60,25 @@ struct common_trie {
|
||||
|
||||
enum match_result { NO_MATCH, PARTIAL_MATCH, COMPLETE_MATCH };
|
||||
|
||||
// Read the symbol at pos. It is a token if a pattern uses the token at pos, else a UTF-8 codepoint.
|
||||
step next(std::string_view sv, const std::vector<int32_t> & token_map, size_t pos) const;
|
||||
|
||||
// Check if a delimiter starts at the given position
|
||||
match_result check_at(std::string_view sv, size_t start_pos) const;
|
||||
match_result check_at(std::string_view sv, const std::vector<int32_t> & token_map, size_t start_pos) const;
|
||||
|
||||
// Insert a word as a sequence of Unicode codepoints, returns its pattern index
|
||||
int32_t insert(const std::string & word);
|
||||
|
||||
// Insert a raw symbol sequence, returns its pattern index (insertion order,
|
||||
// duplicates keep the first index)
|
||||
int32_t insert(const std::vector<uint32_t> & symbols);
|
||||
// Insert a symbol sequence, returns its pattern index (insertion order, duplicates keep the first index)
|
||||
int32_t insert(const std::vector<symbol> & symbols);
|
||||
|
||||
// Child of node on a symbol, created if missing
|
||||
size_t extend(size_t node, symbol ch);
|
||||
|
||||
// Mark node as the end of a pattern with a value from the caller. Returns the value already there, or value if the node had none.
|
||||
// A trie must take its values from mark or from insert, not both.
|
||||
int32_t mark(size_t node, int32_t value);
|
||||
|
||||
private:
|
||||
int32_t n_patterns = 0;
|
||||
@@ -51,11 +92,11 @@ struct common_trie {
|
||||
|
||||
// Aho-Corasick automaton
|
||||
struct common_aho_corasick {
|
||||
common_trie t;
|
||||
std::vector<size_t> fail; // failure links
|
||||
std::vector<size_t> order; // states in BFS order
|
||||
std::vector<int32_t> match; // longest pattern ending at each state (directly or via a suffix link), -1 if none
|
||||
std::set<uint32_t> alphabet; // every character with a transition
|
||||
common_trie t;
|
||||
std::vector<size_t> fail; // failure links
|
||||
std::vector<size_t> order; // states in BFS order
|
||||
std::vector<int32_t> match; // longest pattern ending at each state (directly or via a suffix link), -1 if none
|
||||
std::set<common_trie::symbol> alphabet; // every symbol with a transition
|
||||
|
||||
common_aho_corasick(common_trie trie);
|
||||
|
||||
@@ -69,5 +110,5 @@ struct common_aho_corasick {
|
||||
int32_t match_pattern(size_t s) const { return match[s]; }
|
||||
|
||||
// follow failure links until a transition on `ch` exists.
|
||||
size_t next(size_t state, uint32_t ch) const;
|
||||
size_t next(size_t state, common_trie::symbol ch) const;
|
||||
};
|
||||
@@ -67,6 +67,8 @@ For a more complete example, see `test_example_native()` in
|
||||
- **`end()`** - Matches the end of input (anchor `$`)
|
||||
- **`literal(string)`** - Matches an exact literal string
|
||||
- **`any()`** - Matches any single character (`.`)
|
||||
- **`token(piece)`** - Matches a special token, or the literal `piece` if it is not registered with the builder (see [Special Tokens](#special-tokens))
|
||||
- **`token(pieces, negate)`** - Matches one of the tokens, or with `negate` one token that is not one of them
|
||||
|
||||
### Combinators
|
||||
|
||||
@@ -89,6 +91,8 @@ For a more complete example, see `test_example_native()` in
|
||||
- **`space()`** - Matches zero or more whitespace characters (space, tab, newline)
|
||||
- **`until(delimiter)`** - Matches characters until delimiter is found (delimiter not consumed)
|
||||
- **`until_one_of(delimiters)`** - Matches characters until any delimiter in the list is found
|
||||
- **`until(parser)`** - Matches characters until a delimiter built from literals and tokens is found (delimiter not consumed, see [until(parser)](#untilparser))
|
||||
- **`through(content, branches, optional)`** - Matches content, then the first delimiter found and the rest of its branch (delimiter consumed). The grammar scans for the delimiters instead of following content
|
||||
- **`rest()`** - Matches everything remaining (`.*`)
|
||||
|
||||
### JSON Parsers
|
||||
@@ -108,6 +112,7 @@ For a more complete example, see `test_example_native()` in
|
||||
- **`ref(name)`** - Creates a lightweight reference to a named rule (for recursive grammars)
|
||||
- **`rule(name, p, trigger)`** - Creates a named rule and returns a reference
|
||||
- **`trigger_rule(name, p)`** - Creates a trigger rule (entry point for lazy grammar generation)
|
||||
- **`trigger_rule(name, start, rest)`** - Creates a trigger rule matching `start` then `rest`, gating the lazy grammar on the start
|
||||
- **`schema(p, name, schema, raw)`** - Wraps parser with JSON schema metadata for grammar generation
|
||||
- **`schema(p, name, doc, node, raw)`** - Same, for a node of a `common_chat_schema_document` built earlier, e.g. one tool parameter
|
||||
|
||||
@@ -116,6 +121,110 @@ For a more complete example, see `test_example_native()` in
|
||||
- **`atomic(p)`** - Prevents AST node creation for partial parses
|
||||
- **`tag(tag, p)`** - Creates AST nodes with semantic tags (multiple nodes can share tags)
|
||||
|
||||
## Special Tokens
|
||||
|
||||
Matching a special token by its text alone is ambiguous: a model may emit
|
||||
`<|end|>` as a single control token, or spell it out as plain text, such as
|
||||
when quoting a chat template. The parser can tell them apart when the builder
|
||||
knows the special tokens of the vocabulary, and the input carries the ids of
|
||||
the tokens that produced it.
|
||||
|
||||
A `common_peg_tokens` table constructed from a `llama_vocab` registers every
|
||||
control and user-defined token. Pass it to the builder:
|
||||
|
||||
```cpp
|
||||
common_peg_tokens specials(vocab);
|
||||
|
||||
auto parser = build_chat_peg_parser(specials, [&](common_chat_peg_builder & p) {
|
||||
return p.content(p.until(p.token("<|end|>"))) + p.token("<|end|>");
|
||||
});
|
||||
```
|
||||
|
||||
Chat templates initialized with a model build this table from its vocabulary,
|
||||
and format handlers receive it as `inputs.specials`.
|
||||
|
||||
### token()
|
||||
|
||||
`token(piece)` matches only where the input holds the id of that token, and
|
||||
consumes the whole piece. The same characters without the id do not match.
|
||||
If `piece` is not registered with the builder, it falls back to
|
||||
`literal(piece)`.
|
||||
|
||||
`token(pieces)` matches any of the tokens, and `token(pieces, true)` matches one
|
||||
token that is not in the set. Every piece of a negated set must be registered,
|
||||
otherwise it throws `std::invalid_argument`.
|
||||
|
||||
In the grammar, a token emits `<[id]>` and a negated set emits `!<[...]>`:
|
||||
|
||||
```cpp
|
||||
p.token({ "<a>", "<b>" }) + p.token({ "<a>", "<b>" }, true)
|
||||
```
|
||||
|
||||
```
|
||||
root ::= (<[1]> | <[2]>) !<[1-2]>
|
||||
```
|
||||
|
||||
### until(parser)
|
||||
|
||||
`until(delimiter)` with a parser instead of a string stops at a delimiter that
|
||||
may contain tokens, or several literals joined in a sequence:
|
||||
|
||||
```cpp
|
||||
auto end = p.token("<|end|>");
|
||||
auto final = p.token("<|channel|>") + p.literal("final");
|
||||
|
||||
p.reasoning(p.until(end | final))
|
||||
```
|
||||
|
||||
The delimiter may only be built from:
|
||||
|
||||
- `literal()` and `token()`
|
||||
- `sequence()` and `choice()` of the above
|
||||
- `tag()` and `atomic()` wrapping any of the above
|
||||
|
||||
Any other parser throws `std::invalid_argument`. A choice inside a sequence
|
||||
expands to every combination, so `"a" + ("b" | "d")` stops at both `ab` and
|
||||
`ad`.
|
||||
|
||||
The grammar scans for the delimiters the same way as `until_one_of()`:
|
||||
|
||||
```cpp
|
||||
p.until((p.literal("a") + p.literal("b")) | p.literal("cd"))
|
||||
```
|
||||
|
||||
```
|
||||
root ::= until-5
|
||||
until-5 ::= | [a] until-5-01 | [c] until-5-03 | [^ac] until-5
|
||||
until-5-01 ::= | [a] until-5-01 | [c] until-5-03 | [^abc] until-5
|
||||
until-5-03 ::= | [a] until-5-01 | [c] until-5-03 | [^acd] until-5
|
||||
```
|
||||
|
||||
A token never matches a character rule such as `.` or `[^a]`, so the scan lets
|
||||
the other tokens named elsewhere in the grammar through by name. Here `<b>` is
|
||||
token 2:
|
||||
|
||||
```cpp
|
||||
p.until(p.token("<a>") + p.literal("x")) + p.token("<a>") + p.literal("x") + p.token("<b>")
|
||||
```
|
||||
|
||||
```
|
||||
root ::= until-3 <[1]> "x" <[2]>
|
||||
until-3 ::= | <[1]> until-3-01 | (. | <[2]>) until-3
|
||||
until-3-01 ::= | <[1]> until-3-01 | ([^x] | <[2]>) until-3
|
||||
```
|
||||
|
||||
When the delimiters are made of tokens alone, the scan consumes whole tokens
|
||||
with a negated token set instead:
|
||||
|
||||
```cpp
|
||||
p.until(p.token("<a>")) + p.token("<a>")
|
||||
```
|
||||
|
||||
```
|
||||
root ::= until-1 <[1]>
|
||||
until-1 ::= | !<[1]> until-1
|
||||
```
|
||||
|
||||
## GBNF Grammar Generation
|
||||
|
||||
The PEG parser also acts as a convenient DSL for generating GBNF grammars, with
|
||||
@@ -143,6 +252,28 @@ are emitted in the grammar. All trigger rules are added as alternations in the
|
||||
root rule. It is still necessary to define trigger patterns, as the parser has
|
||||
no interaction with the grammar sampling.
|
||||
|
||||
With `trigger_rule(name, start, rest)`, the grammar instead scans for a start and
|
||||
then constrains the rest. It runs from the first token, so `grammar_lazy` is false
|
||||
and no trigger patterns are needed. For example:
|
||||
|
||||
```cpp
|
||||
auto ab = p.literal("ab");
|
||||
auto cd = p.literal("cd");
|
||||
return p.until(ab | cd) + (p.trigger_rule("one", ab, p.literal("x")) | p.trigger_rule("two", cd, p.literal("y")));
|
||||
```
|
||||
|
||||
```
|
||||
root ::= trigger
|
||||
trigger ::= | [a] trigger-01 | [c] trigger-03 | [^ac] trigger
|
||||
trigger-01 ::= | [b] one-rest | [a] trigger-01 | [c] trigger-03 | [^abc] trigger
|
||||
trigger-03 ::= | [d] two-rest | [a] trigger-01 | [c] trigger-03 | [^acd] trigger
|
||||
one-rest ::= "x"
|
||||
two-rest ::= "y"
|
||||
```
|
||||
|
||||
A start may only be a sequence of literals, tokens, and choices of them, possibly
|
||||
wrapped in tags and atomics.
|
||||
|
||||
### JSON Schema
|
||||
|
||||
The `schema(p, name, schema, raw)` parser will use the `json-schema-to-grammar`
|
||||
|
||||
+4
-2
@@ -73,11 +73,13 @@ Tokens allow grammars to match specific tokenizer tokens rather than character s
|
||||
|
||||
Tokens can be specified in two ways:
|
||||
|
||||
1. **Token ID**: Use angle brackets with the token ID in square brackets: `<[token-id]>`. For example, `<[1000]>` matches the token with ID 1000.
|
||||
1. **Token ID**: Use angle brackets with the token ID in square brackets: `<[token-id]>`. For example, `<[1000]>` matches the token with ID 1000. Multiple IDs and inclusive ranges can be combined: `<[1000,1005-1010]>` matches token 1000 or any token from 1005 to 1010.
|
||||
|
||||
2. **Token string**: Use angle brackets with the token text directly: `<token>`. For example, `<think>` will match the token whose text is exactly `<think>`. This only works if the string tokenizes to exactly one token in the vocabulary, otherwise the grammar will fail to parse.
|
||||
|
||||
You can negate token matches using the `!` prefix: `!<[1000]>` or `!<think>` matches any token *except* the specified one.
|
||||
You can negate token matches using the `!` prefix: `!<[1000]>` or `!<think>` matches any token *except* the specified one, and `!<[1000,1005-1010]>` matches any token outside the set.
|
||||
|
||||
Tokens are matched whole, so a token rule can only match at the start of a token, never after a character rule has consumed part of its text. Any token named by a token rule is also excluded from character rules: once `<think>` appears as a token in a grammar, `.*` in that grammar will not match the `<think>` token.
|
||||
|
||||
```
|
||||
# Match a thinking block: <think>...</think>
|
||||
|
||||
+165
-51
@@ -7,6 +7,7 @@
|
||||
#include <cmath>
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <limits>
|
||||
#include <set>
|
||||
#include <numeric>
|
||||
#include <stdexcept>
|
||||
@@ -33,9 +34,9 @@ static std::pair<uint32_t, const char *> decode_utf8(const char * src) {
|
||||
return std::make_pair(value, pos);
|
||||
}
|
||||
|
||||
static std::pair<std::vector<uint32_t>, llama_partial_utf8> decode_utf8(
|
||||
const std::string & src,
|
||||
llama_partial_utf8 partial_start) {
|
||||
std::pair<std::vector<uint32_t>, llama_partial_utf8> llama_grammar_decode_utf8(
|
||||
const std::string & src,
|
||||
llama_partial_utf8 partial_start) {
|
||||
static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 2, 2, 3, 4 };
|
||||
const char * pos = src.c_str();
|
||||
std::vector<uint32_t> code_points;
|
||||
@@ -185,23 +186,45 @@ static std::pair<uint32_t, const char *> parse_char(const char * src) {
|
||||
throw std::runtime_error("unexpected end of input");
|
||||
}
|
||||
|
||||
static std::pair<uint32_t, const char *> parse_token(const llama_vocab * vocab, const char * src) {
|
||||
static std::pair<uint32_t, const char *> parse_token_id(const char * src) {
|
||||
const char * int_end = parse_int(src);
|
||||
unsigned long id = std::stoul(std::string(src, int_end - src));
|
||||
if (id > std::numeric_limits<uint32_t>::max()) {
|
||||
throw std::runtime_error(std::string("parsed token id is too big at ") + src);
|
||||
}
|
||||
return std::make_pair(static_cast<uint32_t>(id), int_end);
|
||||
}
|
||||
|
||||
static const char * parse_token(const llama_vocab * vocab, const char * src, llama_gretype type, llama_grammar_rule & rule) {
|
||||
const char * pos = src;
|
||||
if (*pos != '<') {
|
||||
throw std::runtime_error(std::string("expecting '<' at ") + pos);
|
||||
}
|
||||
pos++;
|
||||
|
||||
// Parse <[id]>
|
||||
// Parse <[id,id-id,...]>
|
||||
if (*pos == '[') {
|
||||
pos++;
|
||||
const char * int_end = parse_int(pos);
|
||||
unsigned long id = std::stoul(std::string(pos, int_end - pos));
|
||||
if (id > std::numeric_limits<uint32_t>::max()) {
|
||||
throw std::runtime_error(std::string("parsed token id is too big at ") + pos);
|
||||
const size_t start = rule.size();
|
||||
while (true) {
|
||||
auto id_pair = parse_token_id(pos);
|
||||
uint32_t token_id = id_pair.first;
|
||||
pos = id_pair.second;
|
||||
rule.push_back({rule.size() == start ? type : LLAMA_GRETYPE_TOKEN_ALT, token_id});
|
||||
if (*pos == '-') {
|
||||
pos++;
|
||||
auto end_pair = parse_token_id(pos);
|
||||
if (end_pair.first < token_id) {
|
||||
throw std::runtime_error(std::string("invalid token range at ") + pos);
|
||||
}
|
||||
pos = end_pair.second;
|
||||
rule.push_back({LLAMA_GRETYPE_TOKEN_RNG_UPPER, end_pair.first});
|
||||
}
|
||||
if (*pos != ',') {
|
||||
break;
|
||||
}
|
||||
pos++;
|
||||
}
|
||||
uint32_t token_id = static_cast<uint32_t>(id);
|
||||
pos = int_end;
|
||||
if (*pos != ']') {
|
||||
throw std::runtime_error(std::string("expecting ']' at ") + pos);
|
||||
}
|
||||
@@ -210,7 +233,7 @@ static std::pair<uint32_t, const char *> parse_token(const llama_vocab * vocab,
|
||||
throw std::runtime_error(std::string("expecting '>' at ") + pos);
|
||||
}
|
||||
pos++;
|
||||
return std::make_pair(token_id, pos);
|
||||
return pos;
|
||||
}
|
||||
|
||||
if (vocab == nullptr) {
|
||||
@@ -232,7 +255,8 @@ static std::pair<uint32_t, const char *> parse_token(const llama_vocab * vocab,
|
||||
// must tokenize to exactly 1 token
|
||||
throw std::runtime_error("invalid token '" + std::string(src, pos - src) + "'");
|
||||
}
|
||||
return std::make_pair(tokens[0], pos);
|
||||
rule.push_back({type, static_cast<uint32_t>(tokens[0])});
|
||||
return pos;
|
||||
}
|
||||
|
||||
static void print_grammar_char(FILE * file, uint32_t c) {
|
||||
@@ -244,6 +268,16 @@ static void print_grammar_char(FILE * file, uint32_t c) {
|
||||
}
|
||||
}
|
||||
|
||||
static bool is_token_element(llama_grammar_element elem) {
|
||||
switch (elem.type) {
|
||||
case LLAMA_GRETYPE_TOKEN: return true;
|
||||
case LLAMA_GRETYPE_TOKEN_NOT: return true;
|
||||
case LLAMA_GRETYPE_TOKEN_ALT: return true;
|
||||
case LLAMA_GRETYPE_TOKEN_RNG_UPPER: return true;
|
||||
default: return false;
|
||||
}
|
||||
}
|
||||
|
||||
static bool is_char_element(llama_grammar_element elem) {
|
||||
switch (elem.type) {
|
||||
case LLAMA_GRETYPE_CHAR: return true;
|
||||
@@ -258,16 +292,18 @@ static bool is_char_element(llama_grammar_element elem) {
|
||||
static void print_rule_binary(FILE * file, const llama_grammar_rule & rule) {
|
||||
for (auto elem : rule) {
|
||||
switch (elem.type) {
|
||||
case LLAMA_GRETYPE_END: fprintf(file, "END"); break;
|
||||
case LLAMA_GRETYPE_ALT: fprintf(file, "ALT"); break;
|
||||
case LLAMA_GRETYPE_RULE_REF: fprintf(file, "RULE_REF"); break;
|
||||
case LLAMA_GRETYPE_CHAR: fprintf(file, "CHAR"); break;
|
||||
case LLAMA_GRETYPE_CHAR_NOT: fprintf(file, "CHAR_NOT"); break;
|
||||
case LLAMA_GRETYPE_CHAR_RNG_UPPER: fprintf(file, "CHAR_RNG_UPPER"); break;
|
||||
case LLAMA_GRETYPE_CHAR_ALT: fprintf(file, "CHAR_ALT"); break;
|
||||
case LLAMA_GRETYPE_CHAR_ANY: fprintf(file, "CHAR_ANY"); break;
|
||||
case LLAMA_GRETYPE_TOKEN: fprintf(file, "TOKEN"); break;
|
||||
case LLAMA_GRETYPE_TOKEN_NOT: fprintf(file, "TOKEN_NOT"); break;
|
||||
case LLAMA_GRETYPE_END: fprintf(file, "END"); break;
|
||||
case LLAMA_GRETYPE_ALT: fprintf(file, "ALT"); break;
|
||||
case LLAMA_GRETYPE_RULE_REF: fprintf(file, "RULE_REF"); break;
|
||||
case LLAMA_GRETYPE_CHAR: fprintf(file, "CHAR"); break;
|
||||
case LLAMA_GRETYPE_CHAR_NOT: fprintf(file, "CHAR_NOT"); break;
|
||||
case LLAMA_GRETYPE_CHAR_RNG_UPPER: fprintf(file, "CHAR_RNG_UPPER"); break;
|
||||
case LLAMA_GRETYPE_CHAR_ALT: fprintf(file, "CHAR_ALT"); break;
|
||||
case LLAMA_GRETYPE_CHAR_ANY: fprintf(file, "CHAR_ANY"); break;
|
||||
case LLAMA_GRETYPE_TOKEN: fprintf(file, "TOKEN"); break;
|
||||
case LLAMA_GRETYPE_TOKEN_NOT: fprintf(file, "TOKEN_NOT"); break;
|
||||
case LLAMA_GRETYPE_TOKEN_RNG_UPPER: fprintf(file, "TOKEN_RNG_UPPER"); break;
|
||||
case LLAMA_GRETYPE_TOKEN_ALT: fprintf(file, "TOKEN_ALT"); break;
|
||||
}
|
||||
switch (elem.type) {
|
||||
case LLAMA_GRETYPE_END:
|
||||
@@ -295,6 +331,10 @@ static void print_rule_binary(FILE * file, const llama_grammar_rule & rule) {
|
||||
fprintf(file, "%u", elem.value);
|
||||
fprintf(file, "]> ");
|
||||
break;
|
||||
case LLAMA_GRETYPE_TOKEN_RNG_UPPER:
|
||||
case LLAMA_GRETYPE_TOKEN_ALT:
|
||||
fprintf(file, "(%u) ", elem.value);
|
||||
break;
|
||||
}
|
||||
}
|
||||
fprintf(file, "\n");
|
||||
@@ -354,14 +394,32 @@ static void print_rule(
|
||||
case LLAMA_GRETYPE_TOKEN:
|
||||
fprintf(file, "<[");
|
||||
fprintf(file, "%u", elem.value);
|
||||
fprintf(file, "]> ");
|
||||
break;
|
||||
case LLAMA_GRETYPE_TOKEN_NOT:
|
||||
fprintf(file, "!");
|
||||
fprintf(file, "<[");
|
||||
fprintf(file, "%u", elem.value);
|
||||
fprintf(file, "]> ");
|
||||
break;
|
||||
case LLAMA_GRETYPE_TOKEN_RNG_UPPER:
|
||||
if (i == 0 || !is_token_element(rule[i - 1])) {
|
||||
throw std::runtime_error(
|
||||
"LLAMA_GRETYPE_TOKEN_RNG_UPPER without preceding token: " +
|
||||
std::to_string(rule_id) + "," + std::to_string(i));
|
||||
}
|
||||
fprintf(file, "-%u", elem.value);
|
||||
break;
|
||||
case LLAMA_GRETYPE_TOKEN_ALT:
|
||||
if (i == 0 || !is_token_element(rule[i - 1])) {
|
||||
throw std::runtime_error(
|
||||
"LLAMA_GRETYPE_TOKEN_ALT without preceding token: " +
|
||||
std::to_string(rule_id) + "," + std::to_string(i));
|
||||
}
|
||||
fprintf(file, ",%u", elem.value);
|
||||
break;
|
||||
}
|
||||
if (is_token_element(elem) &&
|
||||
rule[i + 1].type != LLAMA_GRETYPE_TOKEN_ALT && rule[i + 1].type != LLAMA_GRETYPE_TOKEN_RNG_UPPER) {
|
||||
fprintf(file, "]> ");
|
||||
}
|
||||
if (is_char_element(elem)) {
|
||||
switch (rule[i + 1].type) {
|
||||
@@ -587,11 +645,9 @@ const char * llama_grammar_parser::parse_sequence(
|
||||
type = LLAMA_GRETYPE_TOKEN_NOT;
|
||||
pos++;
|
||||
}
|
||||
auto token_pair = parse_token(vocab, pos);
|
||||
const char * token_end = token_pair.second;
|
||||
last_sym_start = rule.size();
|
||||
n_prev_rules = 1;
|
||||
rule.push_back({type, token_pair.first});
|
||||
const char * token_end = parse_token(vocab, pos, type, rule);
|
||||
pos = parse_space(token_end, is_nested);
|
||||
} else if (is_word_char(*pos)) { // rule reference
|
||||
const char * name_end = parse_name(pos);
|
||||
@@ -854,17 +910,58 @@ static bool llama_grammar_match_partial_char(
|
||||
|
||||
// returns true iff token matches the rule at pos (regular or inverse)
|
||||
// asserts that pos is pointing to a token element
|
||||
static bool llama_grammar_match_token(
|
||||
static std::pair<bool, const llama_grammar_element *> llama_grammar_match_token(
|
||||
const llama_grammar_element * pos,
|
||||
const llama_token token) {
|
||||
GGML_ASSERT(pos->type == LLAMA_GRETYPE_TOKEN || pos->type == LLAMA_GRETYPE_TOKEN_NOT);
|
||||
if (pos->type == LLAMA_GRETYPE_TOKEN) {
|
||||
return pos->value == static_cast<uint32_t>(token);
|
||||
bool is_positive_token = pos->type == LLAMA_GRETYPE_TOKEN;
|
||||
|
||||
GGML_ASSERT(is_positive_token || pos->type == LLAMA_GRETYPE_TOKEN_NOT);
|
||||
|
||||
const uint32_t id = static_cast<uint32_t>(token);
|
||||
|
||||
bool found = false;
|
||||
do {
|
||||
if (pos[1].type == LLAMA_GRETYPE_TOKEN_RNG_UPPER) {
|
||||
// inclusive range, e.g. <[1-5]>
|
||||
found = found || (pos->value <= id && id <= pos[1].value);
|
||||
pos += 2;
|
||||
} else {
|
||||
found = found || pos->value == id;
|
||||
pos += 1;
|
||||
}
|
||||
} while (pos->type == LLAMA_GRETYPE_TOKEN_ALT);
|
||||
|
||||
return std::make_pair(found == is_positive_token, pos);
|
||||
}
|
||||
|
||||
// every token id a token rule names, merged into sorted inclusive ranges
|
||||
static std::vector<std::pair<uint32_t, uint32_t>> llama_grammar_opaque_tokens(const llama_grammar_rules & rules) {
|
||||
std::vector<std::pair<uint32_t, uint32_t>> ranges;
|
||||
for (const auto & rule : rules) {
|
||||
for (size_t i = 0; i < rule.size(); i++) {
|
||||
const auto type = rule[i].type;
|
||||
if (type == LLAMA_GRETYPE_TOKEN || type == LLAMA_GRETYPE_TOKEN_NOT || type == LLAMA_GRETYPE_TOKEN_ALT) {
|
||||
const bool is_range = i + 1 < rule.size() && rule[i + 1].type == LLAMA_GRETYPE_TOKEN_RNG_UPPER;
|
||||
ranges.emplace_back(rule[i].value, is_range ? rule[i + 1].value : rule[i].value);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (pos->type == LLAMA_GRETYPE_TOKEN_NOT) {
|
||||
return pos->value != static_cast<uint32_t>(token);
|
||||
std::sort(ranges.begin(), ranges.end());
|
||||
std::vector<std::pair<uint32_t, uint32_t>> merged;
|
||||
for (const auto & r : ranges) {
|
||||
if (!merged.empty() && r.first <= (uint64_t) merged.back().second + 1) {
|
||||
merged.back().second = std::max(merged.back().second, r.second);
|
||||
} else {
|
||||
merged.push_back(r);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
return merged;
|
||||
}
|
||||
|
||||
bool llama_grammar_is_opaque(const llama_grammar & grammar, llama_token token) {
|
||||
const auto id = static_cast<uint32_t>(token);
|
||||
auto it = std::upper_bound(grammar.opaque_tokens.begin(), grammar.opaque_tokens.end(), std::make_pair(id, UINT32_MAX));
|
||||
return it != grammar.opaque_tokens.begin() && std::prev(it)->second >= id;
|
||||
}
|
||||
|
||||
// transforms a grammar pushdown stack into N possible stacks, all ending
|
||||
@@ -946,8 +1043,7 @@ static void llama_grammar_advance_stack(
|
||||
break;
|
||||
default:
|
||||
// end of alternate (LLAMA_GRETYPE_END, LLAMA_GRETYPE_ALT) or middle of char range
|
||||
// (LLAMA_GRETYPE_CHAR_ALT, LLAMA_GRETYPE_CHAR_RNG_UPPER); stack should never be left on
|
||||
// those
|
||||
// (LLAMA_GRETYPE_CHAR_ALT, LLAMA_GRETYPE_CHAR_RNG_UPPER) or token set (LLAMA_GRETYPE_TOKEN_ALT, LLAMA_GRETYPE_TOKEN_RNG_UPPER); stack should never be left on those
|
||||
GGML_ABORT("fatal error");
|
||||
}
|
||||
}
|
||||
@@ -1238,13 +1334,13 @@ static bool llama_grammar_reject_step(
|
||||
if (stack_pos->type == LLAMA_GRETYPE_TOKEN || stack_pos->type == LLAMA_GRETYPE_TOKEN_NOT) {
|
||||
for (const uint32_t i : cur) {
|
||||
const auto & tok = candidates[i];
|
||||
if (tok.code_points[depth] == 0) {
|
||||
// reached the end of a token consumed by char rules, reject iff it ended
|
||||
// in a partial response
|
||||
if (tok.partial_utf8.n_remain != 0) {
|
||||
if (depth > 0) {
|
||||
// char rules already consumed part of this token, so it cannot match by id.
|
||||
// accept it only if they consumed all of it and did not end in a partial sequence.
|
||||
if (tok.code_points[depth] != 0 || tok.partial_utf8.n_remain != 0) {
|
||||
rejects.push_back(i);
|
||||
}
|
||||
} else if (!llama_grammar_match_token(stack_pos, tok.id)) {
|
||||
} else if (tok.partial_utf8.n_remain < 0 || !llama_grammar_match_token(stack_pos, tok.id).first) {
|
||||
rejects.push_back(i);
|
||||
}
|
||||
}
|
||||
@@ -1255,7 +1351,10 @@ static bool llama_grammar_reject_step(
|
||||
|
||||
for (const uint32_t i : cur) {
|
||||
const auto & tok = candidates[i];
|
||||
if (tok.code_points[depth] == 0) {
|
||||
if (tok.opaque) {
|
||||
// a token named by a token rule only matches token rules
|
||||
rejects.push_back(i);
|
||||
} else if (tok.code_points[depth] == 0) {
|
||||
// reached end of full codepoints in token, reject iff it ended in a partial sequence
|
||||
// that cannot satisfy this position in grammar
|
||||
if (tok.partial_utf8.n_remain != 0 &&
|
||||
@@ -1425,6 +1524,8 @@ struct llama_grammar * llama_grammar_init_impl(
|
||||
}
|
||||
} while (true);
|
||||
|
||||
auto opaque_tokens = llama_grammar_opaque_tokens(vec_rules);
|
||||
|
||||
// Important: vec_rules has to be moved here, not copied, because stacks contains
|
||||
// pointers to elements of vec_rules. If vec_rules were copied into llama_grammar
|
||||
// then the pointers would be invalidated when the local vec_rules goes out of scope.
|
||||
@@ -1439,6 +1540,7 @@ struct llama_grammar * llama_grammar_init_impl(
|
||||
/* .trigger_buffer_positions = */ {},
|
||||
/* .trigger_tokens = */ {},
|
||||
/* .trigger_patterns = */ {},
|
||||
/* .opaque_tokens = */ std::move(opaque_tokens),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1540,6 +1642,8 @@ struct llama_grammar * llama_grammar_init_impl(
|
||||
trigger.regex = std::regex(trigger.pattern);
|
||||
}
|
||||
|
||||
auto opaque_tokens = llama_grammar_opaque_tokens(vec_rules);
|
||||
|
||||
// Important: vec_rules has to be moved here, not copied, because stacks contains
|
||||
// pointers to elements of vec_rules. If vec_rules were copied into llama_grammar
|
||||
// then the pointers would be invalidated when the local vec_rules goes out of scope.
|
||||
@@ -1554,6 +1658,7 @@ struct llama_grammar * llama_grammar_init_impl(
|
||||
/* .trigger_buffer_positions = */ {},
|
||||
std::move(vec_trigger_tokens),
|
||||
std::move(vec_trigger_patterns),
|
||||
std::move(opaque_tokens),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1601,8 +1706,9 @@ void llama_grammar_apply_impl(const struct llama_grammar & grammar, llama_token_
|
||||
} else if (piece.empty() || piece[0] == 0) {
|
||||
cur_p->data[i].logit = -INFINITY;
|
||||
} else {
|
||||
candidates_decoded.push_back(decode_utf8(piece, grammar.partial_utf8));
|
||||
candidates_grammar.push_back({ i, candidates_decoded.back().first.data(), candidates_decoded.back().second, id });
|
||||
candidates_decoded.push_back(llama_grammar_decode_utf8(piece, grammar.partial_utf8));
|
||||
candidates_grammar.push_back({ i, candidates_decoded.back().first.data(), candidates_decoded.back().second, id,
|
||||
llama_grammar_is_opaque(grammar, id) });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1673,7 +1779,7 @@ void llama_grammar_accept_impl(struct llama_grammar & grammar, llama_token token
|
||||
|
||||
void llama_grammar_accept_str(struct llama_grammar & grammar, const std::string & piece) {
|
||||
// Note terminating 0 in decoded string
|
||||
const auto decoded = decode_utf8(piece, grammar.partial_utf8);
|
||||
const auto decoded = llama_grammar_decode_utf8(piece, grammar.partial_utf8);
|
||||
const auto & code_points = decoded.first;
|
||||
|
||||
for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) {
|
||||
@@ -1688,9 +1794,16 @@ void llama_grammar_accept_str(struct llama_grammar & grammar, const std::string
|
||||
|
||||
void llama_grammar_accept_token(struct llama_grammar & grammar, llama_token token, const std::string & piece) {
|
||||
// Note terminating 0 in decoded string
|
||||
const auto decoded = decode_utf8(piece, grammar.partial_utf8);
|
||||
const auto decoded = llama_grammar_decode_utf8(piece, grammar.partial_utf8);
|
||||
const auto & code_points = decoded.first;
|
||||
|
||||
// the mask rejects a piece that does not continue a partial character, so accepting one has to fail too
|
||||
if (decoded.second.n_remain < 0) {
|
||||
throw std::runtime_error("Invalid UTF-8 continuation in piece: " + piece + " (" + std::to_string(token) + ")");
|
||||
}
|
||||
|
||||
const bool opaque = llama_grammar_is_opaque(grammar, token);
|
||||
|
||||
llama_grammar_stacks stacks_new;
|
||||
stacks_new.reserve(grammar.stacks.size());
|
||||
|
||||
@@ -1702,14 +1815,15 @@ void llama_grammar_accept_token(struct llama_grammar & grammar, llama_token toke
|
||||
const llama_grammar_element * pos = stack.back();
|
||||
|
||||
if (pos->type == LLAMA_GRETYPE_TOKEN || pos->type == LLAMA_GRETYPE_TOKEN_NOT) {
|
||||
if (llama_grammar_match_token(pos, token)) {
|
||||
auto match = llama_grammar_match_token(pos, token);
|
||||
if (match.first) {
|
||||
llama_grammar_stack new_stack(stack.begin(), stack.end() - 1);
|
||||
if (!llama_grammar_is_end_of_sequence(pos + 1)) {
|
||||
new_stack.push_back(pos + 1);
|
||||
if (!llama_grammar_is_end_of_sequence(match.second)) {
|
||||
new_stack.push_back(match.second);
|
||||
}
|
||||
llama_grammar_advance_stack(*grammar.rules, new_stack, stacks_new);
|
||||
}
|
||||
} else {
|
||||
} else if (!opaque) {
|
||||
llama_grammar_stacks current_stacks = {stack};
|
||||
|
||||
for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) {
|
||||
|
||||
+18
-2
@@ -38,11 +38,17 @@ enum llama_gretype {
|
||||
// any character (.)
|
||||
LLAMA_GRETYPE_CHAR_ANY = 7,
|
||||
|
||||
// terminal element: token (<[token-id]>)
|
||||
// terminal element: token (<[token-id]>, <[token-id,token-id-token-id]>)
|
||||
LLAMA_GRETYPE_TOKEN = 8,
|
||||
|
||||
// inverse token (!<[token-id]>)
|
||||
// inverse token (!<[token-id]>, !<[token-id,token-id-token-id]>)
|
||||
LLAMA_GRETYPE_TOKEN_NOT = 9,
|
||||
|
||||
// modifies a preceding LLAMA_GRETYPE_TOKEN, LLAMA_GRETYPE_TOKEN_NOT, or LLAMA_GRETYPE_TOKEN_ALT to be an inclusive range (<[1-5]>)
|
||||
LLAMA_GRETYPE_TOKEN_RNG_UPPER = 10,
|
||||
|
||||
// modifies a preceding LLAMA_GRETYPE_TOKEN, LLAMA_GRETYPE_TOKEN_NOT, or LLAMA_GRETYPE_TOKEN_RNG_UPPER to add an alternate token to match (<[1,5]>, <[1-5,9]>)
|
||||
LLAMA_GRETYPE_TOKEN_ALT = 11,
|
||||
};
|
||||
|
||||
typedef struct llama_grammar_element {
|
||||
@@ -60,6 +66,7 @@ struct llama_grammar_candidate {
|
||||
const uint32_t * code_points;
|
||||
llama_partial_utf8 partial_utf8;
|
||||
llama_token id;
|
||||
bool opaque;
|
||||
};
|
||||
|
||||
using llama_grammar_rule = std::vector< llama_grammar_element>;
|
||||
@@ -84,6 +91,10 @@ std::vector<llama_grammar_candidate> llama_grammar_reject_candidates_for_stack(
|
||||
const llama_grammar_stack & stack,
|
||||
const llama_grammar_candidates & candidates);
|
||||
|
||||
std::pair<std::vector<uint32_t>, llama_partial_utf8> llama_grammar_decode_utf8(
|
||||
const std::string & src,
|
||||
llama_partial_utf8 partial_start);
|
||||
|
||||
struct llama_grammar_parser {
|
||||
const llama_vocab * vocab;
|
||||
std::map<std::string, uint32_t> symbol_ids;
|
||||
@@ -151,6 +162,8 @@ struct llama_grammar {
|
||||
trigger_patterns; // Regular expressions that trigger a lazy grammar. Must be a full match of the entire generated
|
||||
// string, and the grammar will be given the string from the first match group onwards.
|
||||
|
||||
// token ids named by token rules, as sorted inclusive ranges. char rules never match these tokens.
|
||||
std::vector<std::pair<uint32_t, uint32_t>> opaque_tokens;
|
||||
};
|
||||
|
||||
//
|
||||
@@ -195,3 +208,6 @@ void llama_grammar_accept_token(
|
||||
struct llama_grammar & grammar,
|
||||
llama_token token,
|
||||
const std::string & piece);
|
||||
|
||||
// true if a token rule names the token, char rules do not match it
|
||||
bool llama_grammar_is_opaque(const llama_grammar & grammar, llama_token token);
|
||||
@@ -367,6 +367,49 @@ void test_basic(testing & t) {
|
||||
});
|
||||
|
||||
|
||||
t.test("through", [](testing & t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.through(p.tag("body", p.until_one_of({ "</a>", "</b>" })), { { p.literal("</a>"), p.literal("x") }, { p.literal("</b>") } }, true);
|
||||
});
|
||||
|
||||
t.test("consumes the delimiter and its rest", [&](testing & t) {
|
||||
auto ctx = common_peg_parse_context("hello</a>x");
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_equal("success", true, result.success());
|
||||
t.assert_equal("end", (size_t) 10, result.end);
|
||||
t.assert_equal("body", std::string("hello"), std::string(ctx.ast.get(result.nodes[0]).text));
|
||||
});
|
||||
|
||||
t.test("picks the branch of the delimiter found", [&](testing & t) {
|
||||
auto ctx = common_peg_parse_context("hello</b>x");
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_equal("success", true, result.success());
|
||||
t.assert_equal("end", (size_t) 9, result.end);
|
||||
});
|
||||
|
||||
t.test("waits for the rest of a partial delimiter", [&](testing & t) {
|
||||
auto ctx = common_peg_parse_context("hello</", COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_equal("need_more_input", true, result.need_more_input());
|
||||
});
|
||||
|
||||
t.test("optional may end before a delimiter", [&](testing & t) {
|
||||
auto ctx = common_peg_parse_context("hello");
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_equal("success", true, result.success());
|
||||
t.assert_equal("end", (size_t) 5, result.end);
|
||||
});
|
||||
|
||||
t.test("fails when content stops off a delimiter", [&](testing & t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.through(p.literal("he"), "</a>");
|
||||
});
|
||||
auto ctx = common_peg_parse_context("hello</a>");
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_equal("fail", true, result.fail());
|
||||
});
|
||||
});
|
||||
|
||||
t.test("recursive rules", [](testing &t) {
|
||||
// Test simple number
|
||||
t.test("simple_number", [](testing &t) {
|
||||
|
||||
@@ -129,14 +129,14 @@ void test_gbnf_generation(testing &t) {
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= until-0
|
||||
root ::= until-1
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
until-0 ::= | [<] until-0-01 | [^<] until-0
|
||||
until-0-01 ::= | [<] until-0-01 | [/] until-0-02 | [^/<] until-0
|
||||
until-0-02 ::= | [<] until-0-01 | [t] until-0-03 | [^<t] until-0
|
||||
until-0-03 ::= | [<] until-0-01 | [a] until-0-04 | [^<a] until-0
|
||||
until-0-04 ::= | [<] until-0-01 | [g] until-0-05 | [^<g] until-0
|
||||
until-0-05 ::= | [<] until-0-01 | [^<>] until-0
|
||||
until-1 ::= | [<] until-1-01 | [^<] until-1
|
||||
until-1-01 ::= | [<] until-1-01 | [/] until-1-02 | [^/<] until-1
|
||||
until-1-02 ::= | [<] until-1-01 | [t] until-1-03 | [^<t] until-1
|
||||
until-1-03 ::= | [<] until-1-01 | [a] until-1-04 | [^<a] until-1
|
||||
until-1-04 ::= | [<] until-1-01 | [g] until-1-05 | [^<g] until-1
|
||||
until-1-05 ::= | [<] until-1-01 | [^<>] until-1
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
@@ -150,22 +150,22 @@ void test_gbnf_generation(testing &t) {
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= until-0
|
||||
root ::= until-1
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
until-0 ::= | [\n] until-0-01 | [^\n] until-0
|
||||
until-0-01 ::= | [\n] until-0-01 | [<] until-0-02 | [^\n<] until-0
|
||||
until-0-02 ::= | [\n] until-0-01 | [/] until-0-03 | [^\n/] until-0
|
||||
until-0-03 ::= | [\n] until-0-01 | [p] until-0-04 | [^\np] until-0
|
||||
until-0-04 ::= | [\n] until-0-01 | [a] until-0-05 | [^\na] until-0
|
||||
until-0-05 ::= | [\n] until-0-01 | [r] until-0-06 | [^\nr] until-0
|
||||
until-0-06 ::= | [\n] until-0-01 | [a] until-0-07 | [^\na] until-0
|
||||
until-0-07 ::= | [\n] until-0-01 | [m] until-0-08 | [^\nm] until-0
|
||||
until-0-08 ::= | [\n] until-0-01 | [e] until-0-09 | [^\ne] until-0
|
||||
until-0-09 ::= | [\n] until-0-01 | [t] until-0-10 | [^\nt] until-0
|
||||
until-0-10 ::= | [\n] until-0-01 | [e] until-0-11 | [^\ne] until-0
|
||||
until-0-11 ::= | [\n] until-0-01 | [r] until-0-12 | [^\nr] until-0
|
||||
until-0-12 ::= | [\n] until-0-01 | [>] until-0-13 | [^\n>] until-0
|
||||
until-0-13 ::= | [^\n] until-0
|
||||
until-1 ::= | [\n] until-1-01 | [^\n] until-1
|
||||
until-1-01 ::= | [\n] until-1-01 | [<] until-1-02 | [^\n<] until-1
|
||||
until-1-02 ::= | [\n] until-1-01 | [/] until-1-03 | [^\n/] until-1
|
||||
until-1-03 ::= | [\n] until-1-01 | [p] until-1-04 | [^\np] until-1
|
||||
until-1-04 ::= | [\n] until-1-01 | [a] until-1-05 | [^\na] until-1
|
||||
until-1-05 ::= | [\n] until-1-01 | [r] until-1-06 | [^\nr] until-1
|
||||
until-1-06 ::= | [\n] until-1-01 | [a] until-1-07 | [^\na] until-1
|
||||
until-1-07 ::= | [\n] until-1-01 | [m] until-1-08 | [^\nm] until-1
|
||||
until-1-08 ::= | [\n] until-1-01 | [e] until-1-09 | [^\ne] until-1
|
||||
until-1-09 ::= | [\n] until-1-01 | [t] until-1-10 | [^\nt] until-1
|
||||
until-1-10 ::= | [\n] until-1-01 | [e] until-1-11 | [^\ne] until-1
|
||||
until-1-11 ::= | [\n] until-1-01 | [r] until-1-12 | [^\nr] until-1
|
||||
until-1-12 ::= | [\n] until-1-01 | [>] until-1-13 | [^\n>] until-1
|
||||
until-1-13 ::= | [^\n] until-1
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
@@ -181,15 +181,15 @@ void test_gbnf_generation(testing &t) {
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= until-0
|
||||
root ::= until-1
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
until-0 ::= | [<] until-0-01 | [^<] until-0
|
||||
until-0-01 ::= | [<] until-0-01 | [\uFF5C] until-0-02 | [^<\uFF5C] until-0
|
||||
until-0-02 ::= | [<] until-0-01 | [D] until-0-03 | [^<D] until-0
|
||||
until-0-03 ::= | [<] until-0-01 | [S] until-0-04 | [^<S] until-0
|
||||
until-0-04 ::= | [<] until-0-01 | [M] until-0-05 | [^<M] until-0
|
||||
until-0-05 ::= | [<] until-0-01 | [L] until-0-06 | [^<L] until-0
|
||||
until-0-06 ::= | [<] until-0-01 | [^<\uFF5C] until-0
|
||||
until-1 ::= | [<] until-1-01 | [^<] until-1
|
||||
until-1-01 ::= | [<] until-1-01 | [\uFF5C] until-1-02 | [^<\uFF5C] until-1
|
||||
until-1-02 ::= | [<] until-1-01 | [D] until-1-03 | [^<D] until-1
|
||||
until-1-03 ::= | [<] until-1-01 | [S] until-1-04 | [^<S] until-1
|
||||
until-1-04 ::= | [<] until-1-01 | [M] until-1-05 | [^<M] until-1
|
||||
until-1-05 ::= | [<] until-1-01 | [L] until-1-06 | [^<L] until-1
|
||||
until-1-06 ::= | [<] until-1-01 | [^<\uFF5C] until-1
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
@@ -203,18 +203,18 @@ void test_gbnf_generation(testing &t) {
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= until-0
|
||||
root ::= until-3
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
until-0 ::= | [a] until-0-01 | [c] until-0-03 | [e] until-0-05 | [^ace] until-0
|
||||
until-0-01 ::= | [a] until-0-01 | [c] until-0-03 | [e] until-0-05 | [^abce] until-0
|
||||
until-0-03 ::= | [a] until-0-01 | [c] until-0-03 | [e] until-0-05 | [^acde] until-0
|
||||
until-0-05 ::= | [a] until-0-01 | [c] until-0-03 | [e] until-0-05 | [^acef] until-0
|
||||
until-3 ::= | [a] until-3-01 | [c] until-3-03 | [e] until-3-05 | [^ace] until-3
|
||||
until-3-01 ::= | [a] until-3-01 | [c] until-3-03 | [e] until-3-05 | [^abce] until-3
|
||||
until-3-03 ::= | [a] until-3-01 | [c] until-3-03 | [e] until-3-05 | [^acde] until-3
|
||||
until-3-05 ::= | [a] until-3-01 | [c] until-3-03 | [e] until-3-05 | [^acef] until-3
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("ac grammar", [](testing &t) {
|
||||
t.test("through grammar", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.ac(p.until("</tag>") + p.literal("</tag>"), "</tag>");
|
||||
return p.through(p.until("</tag>"), "</tag>");
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
@@ -222,20 +222,20 @@ void test_gbnf_generation(testing &t) {
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
ac-3 ::= [<] ac-3-01 | [^<] ac-3
|
||||
ac-3-01 ::= [<] ac-3-01 | [/] ac-3-02 | [^/<] ac-3
|
||||
ac-3-02 ::= [<] ac-3-01 | [t] ac-3-03 | [^<t] ac-3
|
||||
ac-3-03 ::= [<] ac-3-01 | [a] ac-3-04 | [^<a] ac-3
|
||||
ac-3-04 ::= [<] ac-3-01 | [g] ac-3-05 | [^<g] ac-3
|
||||
ac-3-05 ::= [>] | [<] ac-3-01 | [^<>] ac-3
|
||||
root ::= ac-3
|
||||
root ::= until-3
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
until-3 ::= [<] until-3-01 | [^<] until-3
|
||||
until-3-01 ::= [<] until-3-01 | [/] until-3-02 | [^/<] until-3
|
||||
until-3-02 ::= [<] until-3-01 | [t] until-3-03 | [^<t] until-3
|
||||
until-3-03 ::= [<] until-3-01 | [a] until-3-04 | [^<a] until-3
|
||||
until-3-04 ::= [<] until-3-01 | [g] until-3-05 | [^<g] until-3
|
||||
until-3-05 ::= [>] | [<] until-3-01 | [^<>] until-3
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("ac grammar terminates at first delimiter", [](testing &t) {
|
||||
t.test("through grammar terminates at first delimiter", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.ac(p.until("\n</parameter>\n") + p.literal("\n</parameter>\n"), "\n</parameter>\n");
|
||||
return p.through(p.until("\n</parameter>\n"), "\n</parameter>\n");
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
@@ -243,28 +243,28 @@ void test_gbnf_generation(testing &t) {
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
ac-3 ::= [\n] ac-3-01 | [^\n] ac-3
|
||||
ac-3-01 ::= [\n] ac-3-01 | [<] ac-3-02 | [^\n<] ac-3
|
||||
ac-3-02 ::= [\n] ac-3-01 | [/] ac-3-03 | [^\n/] ac-3
|
||||
ac-3-03 ::= [\n] ac-3-01 | [p] ac-3-04 | [^\np] ac-3
|
||||
ac-3-04 ::= [\n] ac-3-01 | [a] ac-3-05 | [^\na] ac-3
|
||||
ac-3-05 ::= [\n] ac-3-01 | [r] ac-3-06 | [^\nr] ac-3
|
||||
ac-3-06 ::= [\n] ac-3-01 | [a] ac-3-07 | [^\na] ac-3
|
||||
ac-3-07 ::= [\n] ac-3-01 | [m] ac-3-08 | [^\nm] ac-3
|
||||
ac-3-08 ::= [\n] ac-3-01 | [e] ac-3-09 | [^\ne] ac-3
|
||||
ac-3-09 ::= [\n] ac-3-01 | [t] ac-3-10 | [^\nt] ac-3
|
||||
ac-3-10 ::= [\n] ac-3-01 | [e] ac-3-11 | [^\ne] ac-3
|
||||
ac-3-11 ::= [\n] ac-3-01 | [r] ac-3-12 | [^\nr] ac-3
|
||||
ac-3-12 ::= [\n] ac-3-01 | [>] ac-3-13 | [^\n>] ac-3
|
||||
ac-3-13 ::= [\n] | [^\n] ac-3
|
||||
root ::= ac-3
|
||||
root ::= until-3
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
until-3 ::= [\n] until-3-01 | [^\n] until-3
|
||||
until-3-01 ::= [\n] until-3-01 | [<] until-3-02 | [^\n<] until-3
|
||||
until-3-02 ::= [\n] until-3-01 | [/] until-3-03 | [^\n/] until-3
|
||||
until-3-03 ::= [\n] until-3-01 | [p] until-3-04 | [^\np] until-3
|
||||
until-3-04 ::= [\n] until-3-01 | [a] until-3-05 | [^\na] until-3
|
||||
until-3-05 ::= [\n] until-3-01 | [r] until-3-06 | [^\nr] until-3
|
||||
until-3-06 ::= [\n] until-3-01 | [a] until-3-07 | [^\na] until-3
|
||||
until-3-07 ::= [\n] until-3-01 | [m] until-3-08 | [^\nm] until-3
|
||||
until-3-08 ::= [\n] until-3-01 | [e] until-3-09 | [^\ne] until-3
|
||||
until-3-09 ::= [\n] until-3-01 | [t] until-3-10 | [^\nt] until-3
|
||||
until-3-10 ::= [\n] until-3-01 | [e] until-3-11 | [^\ne] until-3
|
||||
until-3-11 ::= [\n] until-3-01 | [r] until-3-12 | [^\nr] until-3
|
||||
until-3-12 ::= [\n] until-3-01 | [>] until-3-13 | [^\n>] until-3
|
||||
until-3-13 ::= [\n] | [^\n] until-3
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("ac grammar multiple delimiters", [](testing &t) {
|
||||
t.test("through grammar multiple delimiters", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.ac(p.eps(), std::vector<std::string>{"ab", "cd", "ef"});
|
||||
return p.through(p.eps(), { { p.literal("ab") }, { p.literal("cd") }, { p.literal("ef") } });
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
@@ -272,12 +272,238 @@ void test_gbnf_generation(testing &t) {
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
ac-1 ::= [a] ac-1-01 | [c] ac-1-03 | [e] ac-1-05 | [^ace] ac-1
|
||||
ac-1-01 ::= [b] | [a] ac-1-01 | [c] ac-1-03 | [e] ac-1-05 | [^abce] ac-1
|
||||
ac-1-03 ::= [d] | [a] ac-1-01 | [c] ac-1-03 | [e] ac-1-05 | [^acde] ac-1
|
||||
ac-1-05 ::= [f] | [a] ac-1-01 | [c] ac-1-03 | [e] ac-1-05 | [^acef] ac-1
|
||||
root ::= ac-1
|
||||
root ::= until-4
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
until-4 ::= [a] until-4-01 | [c] until-4-03 | [e] until-4-05 | [^ace] until-4
|
||||
until-4-01 ::= [b] | [a] until-4-01 | [c] until-4-03 | [e] until-4-05 | [^abce] until-4
|
||||
until-4-03 ::= [d] | [a] until-4-01 | [c] until-4-03 | [e] until-4-05 | [^acde] until-4
|
||||
until-4-05 ::= [f] | [a] until-4-01 | [c] until-4-03 | [e] until-4-05 | [^acef] until-4
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("until grammar with parser delimiters", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.until((p.literal("a") + p.literal("b")) | p.literal("cd"));
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= until-5
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
until-5 ::= | [a] until-5-01 | [c] until-5-03 | [^ac] until-5
|
||||
until-5-01 ::= | [a] until-5-01 | [c] until-5-03 | [^abc] until-5
|
||||
until-5-03 ::= | [a] until-5-01 | [c] until-5-03 | [^acd] until-5
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("delimiters see through tags and atomics", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.until(p.tag("open", p.literal("a") + p.atomic(p.literal("b"))));
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= until-5
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
until-5 ::= | [a] until-5-01 | [^a] until-5
|
||||
until-5-01 ::= | [a] until-5-01 | [^ab] until-5
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("delimiters expand a choice in a sequence", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.until(p.literal("a") + (p.literal("b") | p.literal("d")));
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= until-5
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
until-5 ::= | [a] until-5-01 | [^a] until-5
|
||||
until-5-01 ::= | [a] until-5-01 | [^abd] until-5
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("delimiters reject anything but sequences, choices, literals, and tokens", [](testing &t) {
|
||||
bool threw = false;
|
||||
try {
|
||||
build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.until(p.literal("a") + p.space());
|
||||
});
|
||||
} catch (const std::invalid_argument &) {
|
||||
threw = true;
|
||||
}
|
||||
t.assert_equal("space_rejected", true, threw);
|
||||
});
|
||||
|
||||
t.test("through branches reject a shared delimiter", [](testing &t) {
|
||||
bool threw = false;
|
||||
try {
|
||||
build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.through(p.eps(), { { p.literal("ab"), p.literal("x") }, { p.literal("ab"), p.literal("y") } });
|
||||
});
|
||||
} catch (const std::invalid_argument &) {
|
||||
threw = true;
|
||||
}
|
||||
t.assert_equal("duplicate_rejected", true, threw);
|
||||
});
|
||||
|
||||
t.test("through grammar branches continue with their rest", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.through(p.eps(), { { p.literal("ab"), p.literal("x") }, { p.literal("cd") } });
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= until-4
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
until-4 ::= [a] until-4-01 | [c] until-4-03 | [^ac] until-4
|
||||
until-4-01 ::= [b] until-4-rest-0 | [a] until-4-01 | [c] until-4-03 | [^abc] until-4
|
||||
until-4-03 ::= [d] | [a] until-4-01 | [c] until-4-03 | [^acd] until-4
|
||||
until-4-rest-0 ::= "x"
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("through grammar optional may end before a delimiter", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.through(p.eps(), { { p.literal("ab"), p.literal("x") } }, true);
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= until-3
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
until-3 ::= | [a] until-3-01 | [^a] until-3
|
||||
until-3-01 ::= | [b] until-3-rest-0 | [a] until-3-01 | [^ab] until-3
|
||||
until-3-rest-0 ::= "x"
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("trigger rules emit their start and rest when not lazy", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
auto ab = p.literal("ab");
|
||||
auto cd = p.literal("cd");
|
||||
auto scan = p.until(ab | cd);
|
||||
return scan + (p.trigger_rule("one", ab, p.literal("x")) | p.trigger_rule("two", cd, p.literal("y")));
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder, false);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
one ::= "ab" "x"
|
||||
root ::= until-3 (one | two)
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
two ::= "cd" "y"
|
||||
until-3 ::= | [a] until-3-01 | [c] until-3-03 | [^ac] until-3
|
||||
until-3-01 ::= | [a] until-3-01 | [c] until-3-03 | [^abc] until-3
|
||||
until-3-03 ::= | [a] until-3-01 | [c] until-3-03 | [^acd] until-3
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("trigger rules make the lazy grammar scan for their starts", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
auto ab = p.literal("ab");
|
||||
auto cd = p.literal("cd");
|
||||
auto scan = p.until(ab | cd);
|
||||
return scan + (p.trigger_rule("one", ab, p.literal("x")) | p.trigger_rule("two", cd, p.literal("y")));
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder, true);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
one-rest ::= "x"
|
||||
root ::= trigger
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
trigger ::= | [a] trigger-01 | [c] trigger-03 | [^ac] trigger
|
||||
trigger-01 ::= | [b] one-rest | [a] trigger-01 | [c] trigger-03 | [^abc] trigger
|
||||
trigger-03 ::= | [d] two-rest | [a] trigger-01 | [c] trigger-03 | [^acd] trigger
|
||||
two-rest ::= "y"
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("scanning grammar lets other named tokens through", [](testing &t) {
|
||||
common_peg_tokens tokens;
|
||||
tokens.ids.emplace("<a>", 1);
|
||||
tokens.ids.emplace("<b>", 2);
|
||||
tokens.tokens.emplace(1, common_peg_token{ "<a>", LLAMA_TOKEN_ATTR_CONTROL });
|
||||
tokens.tokens.emplace(2, common_peg_token{ "<b>", LLAMA_TOKEN_ATTR_CONTROL });
|
||||
|
||||
common_peg_parser_builder p(tokens);
|
||||
auto start = p.token("<a>") + p.literal("x");
|
||||
p.set_root(p.until(start) + p.trigger_rule("tool", start, p.token("<b>")));
|
||||
auto parser = p.build();
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder, true);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= trigger
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
tool-rest ::= <[2]>
|
||||
trigger ::= | <[1]> trigger-01 | (. | <[2]>) trigger
|
||||
trigger-01 ::= | [x] tool-rest | <[1]> trigger-01 | ([^x] | <[2]>) trigger
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("token sets", [](testing &t) {
|
||||
common_peg_tokens tokens;
|
||||
tokens.ids.emplace("<a>", 1);
|
||||
tokens.ids.emplace("<b>", 2);
|
||||
tokens.tokens.emplace(1, common_peg_token{ "<a>", LLAMA_TOKEN_ATTR_CONTROL });
|
||||
tokens.tokens.emplace(2, common_peg_token{ "<b>", LLAMA_TOKEN_ATTR_CONTROL });
|
||||
|
||||
common_peg_parser_builder p(tokens);
|
||||
p.set_root(p.token({ "<a>", "<b>" }) + p.token({ "<a>", "<b>" }, true));
|
||||
auto parser = p.build();
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= (<[1]> | <[2]>) !<[1-2]>
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("through grammar suffix delimiter offers both rests", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.through(p.eps(), { { p.literal("xab"), p.literal("1") }, { p.literal("ab"), p.literal("2") } });
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= until-5
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
until-5 ::= [x] until-5-01 | [a] until-5-04 | [^ax] until-5
|
||||
until-5-01 ::= [x] until-5-01 | [a] until-5-02 | [^ax] until-5
|
||||
until-5-02 ::= [b] (until-5-rest-0 | until-5-rest-1) | [x] until-5-01 | [a] until-5-04 | [^abx] until-5
|
||||
until-5-04 ::= [b] until-5-rest-1 | [x] until-5-01 | [a] until-5-04 | [^abx] until-5
|
||||
until-5-rest-0 ::= "1"
|
||||
until-5-rest-1 ::= "2"
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
|
||||
+225
-48
@@ -305,6 +305,66 @@ static std::string get_expected_description(const llama_grammar_rules & rules, c
|
||||
return result;
|
||||
}
|
||||
|
||||
// Every text token shares one ID, the parsers and grammars only tell special tokens apart
|
||||
static constexpr llama_token FAKE_TEXT_TOKEN = 0;
|
||||
|
||||
struct fake_token {
|
||||
llama_token id;
|
||||
std::string piece;
|
||||
};
|
||||
|
||||
// Splits text into the special tokens it spells, longest first, and one text token per UTF-8 character.
|
||||
static std::vector<fake_token> fake_tokenize(const common_peg_tokens & specials, const std::string & text) {
|
||||
std::vector<fake_token> result;
|
||||
size_t pos = 0;
|
||||
while (pos < text.size()) {
|
||||
llama_token best = LLAMA_TOKEN_NULL;
|
||||
size_t best_len = 0;
|
||||
for (const auto & [piece, id] : specials.ids) {
|
||||
if (piece.size() > best_len && text.compare(pos, piece.size(), piece) == 0) {
|
||||
best = id;
|
||||
best_len = piece.size();
|
||||
}
|
||||
}
|
||||
if (best != LLAMA_TOKEN_NULL) {
|
||||
result.push_back({ best, text.substr(pos, best_len) });
|
||||
pos += best_len;
|
||||
continue;
|
||||
}
|
||||
size_t len = 1;
|
||||
while (pos + len < text.size() && (static_cast<unsigned char>(text[pos + len]) & 0xC0) == 0x80) {
|
||||
len++;
|
||||
}
|
||||
result.push_back({ FAKE_TEXT_TOKEN, text.substr(pos, len) });
|
||||
pos += len;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// Returns the tokens from byte offset pos. A token split by pos keeps the rest of its piece as text.
|
||||
static std::vector<fake_token> fake_tokens_from(const std::vector<fake_token> & tokens, size_t pos) {
|
||||
std::vector<fake_token> result;
|
||||
size_t start = 0;
|
||||
for (const auto & token : tokens) {
|
||||
const size_t end = start + token.piece.size();
|
||||
if (start >= pos) {
|
||||
result.push_back(token);
|
||||
} else if (end > pos) {
|
||||
result.push_back({ FAKE_TEXT_TOKEN, token.piece.substr(pos - start) });
|
||||
}
|
||||
start = end;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static common_chat_input fake_input(const std::vector<fake_token> & tokens) {
|
||||
common_chat_input input;
|
||||
for (const auto & token : tokens) {
|
||||
input.append(token.piece, token.id);
|
||||
}
|
||||
return input;
|
||||
}
|
||||
|
||||
// Result of a detailed grammar match attempt
|
||||
struct grammar_match_result {
|
||||
bool success = false; // Did the string fully match the grammar?
|
||||
@@ -318,56 +378,49 @@ struct grammar_match_result {
|
||||
bool incomplete = false; // True if matched all input but grammar expects more
|
||||
};
|
||||
|
||||
// Detailed version of match_string that returns failure information
|
||||
static grammar_match_result match_string_detailed(const std::string & input, llama_grammar * grammar) {
|
||||
grammar_match_result result;
|
||||
result.total_bytes = input.size();
|
||||
// Detailed version of match_string that returns failure information, accepting the input token by token
|
||||
static grammar_match_result match_string_detailed(const std::vector<fake_token> & tokens, llama_grammar * grammar) {
|
||||
std::string input;
|
||||
for (const auto & token : tokens) {
|
||||
input += token.piece;
|
||||
}
|
||||
|
||||
const auto cpts = unicode_cpts_from_utf8(input);
|
||||
result.total_codepoints = cpts.size();
|
||||
grammar_match_result result;
|
||||
result.total_bytes = input.size();
|
||||
result.total_codepoints = unicode_cpts_from_utf8(input).size();
|
||||
|
||||
auto & stacks_cur = llama_grammar_get_stacks(grammar);
|
||||
const auto & rules = llama_grammar_get_rules(grammar);
|
||||
|
||||
size_t byte_pos = 0;
|
||||
size_t cpt_pos = 0;
|
||||
|
||||
for (size_t i = 0; i < cpts.size(); i++) {
|
||||
const auto & cpt = cpts[i];
|
||||
|
||||
for (const auto & token : tokens) {
|
||||
// Get expected before accepting (for error reporting)
|
||||
std::string expected_before = get_expected_description(rules, stacks_cur);
|
||||
|
||||
llama_grammar_accept(grammar, cpt);
|
||||
|
||||
// Calculate byte position for this codepoint
|
||||
size_t cpt_bytes = 0;
|
||||
if (cpt < 0x80) {
|
||||
cpt_bytes = 1;
|
||||
} else if (cpt < 0x800) {
|
||||
cpt_bytes = 2;
|
||||
} else if (cpt < 0x10000) {
|
||||
cpt_bytes = 3;
|
||||
} else {
|
||||
cpt_bytes = 4;
|
||||
}
|
||||
|
||||
if (stacks_cur.empty()) {
|
||||
try {
|
||||
llama_grammar_accept_token(*grammar, token.id, token.piece);
|
||||
} catch (const std::runtime_error & /*e*/) {
|
||||
// Grammar failed to match at this point
|
||||
result.matched_bytes = byte_pos;
|
||||
result.matched_codepoints = i;
|
||||
result.matched_codepoints = cpt_pos;
|
||||
result.matched_prefix = input.substr(0, byte_pos);
|
||||
result.failing_char = format_codepoint(cpt);
|
||||
result.failing_char = token.id == FAKE_TEXT_TOKEN
|
||||
? format_codepoint(unicode_cpts_from_utf8(token.piece).at(0))
|
||||
: token.piece + " (token " + std::to_string(token.id) + ")";
|
||||
result.expected_description = expected_before;
|
||||
result.incomplete = false;
|
||||
return result;
|
||||
}
|
||||
|
||||
byte_pos += cpt_bytes;
|
||||
byte_pos += token.piece.size();
|
||||
cpt_pos += unicode_cpts_from_utf8(token.piece).size();
|
||||
}
|
||||
|
||||
// All input matched - check if grammar is complete
|
||||
result.matched_bytes = input.size();
|
||||
result.matched_codepoints = cpts.size();
|
||||
result.matched_codepoints = result.total_codepoints;
|
||||
result.matched_prefix = input;
|
||||
|
||||
if (std::any_of(stacks_cur.begin(), stacks_cur.end(), [](const auto & stack) { return stack.empty(); })) {
|
||||
@@ -1129,6 +1182,7 @@ static std::string g_template_filter;
|
||||
static bool g_force_reconstruction_test = false;
|
||||
|
||||
static void test_peg_parser(common_chat_templates * tmpls,
|
||||
const common_peg_tokens & specials,
|
||||
const std::function<void(peg_test_case &)> & init,
|
||||
bool detailed_debug) {
|
||||
// UTF-8-safe truncation helper (same as in test_parser_with_streaming)
|
||||
@@ -1173,8 +1227,15 @@ static void test_peg_parser(common_chat_templates * tmpls,
|
||||
tc.expect.role = "assistant";
|
||||
}
|
||||
|
||||
const bool tokenized = !specials.ids.empty();
|
||||
|
||||
common_chat_session_params session_params;
|
||||
session_params.debug = detailed_debug;
|
||||
if (tokenized) {
|
||||
session_params.tokenize = [&](const std::string & text) {
|
||||
return fake_input(fake_tokenize(specials, text));
|
||||
};
|
||||
}
|
||||
common_chat_session session(tmpls, nullptr, tc.params, session_params);
|
||||
const auto & parser = session.parser();
|
||||
common_params_sampling sampling;
|
||||
@@ -1200,14 +1261,27 @@ static void test_peg_parser(common_chat_templates * tmpls,
|
||||
common_chat_msg msg_prev;
|
||||
msg_accum.role = msg_prev.role = "assistant";
|
||||
|
||||
size_t fed = 0;
|
||||
for (size_t i = 1; i <= tc.input.size(); ++i) {
|
||||
auto is_partial = i < tc.input.size() || tc.is_partial;
|
||||
// Use UTF-8 safe truncation to avoid corrupting multi-byte characters
|
||||
size_t safe_len = utf8_truncate_safe_len(std::string_view(tc.input).substr(0, i));
|
||||
common_chat_input chunk(tc.input.substr(fed, safe_len - fed));
|
||||
fed = safe_len;
|
||||
const common_chat_msg & msg_current = is_partial ? session.feed(chunk) : session.finish(chunk);
|
||||
const auto input_tokens = fake_tokenize(specials, tc.input);
|
||||
|
||||
// A token arrives whole, so a special token is never split across chunks
|
||||
std::vector<common_chat_input> chunks;
|
||||
if (tokenized) {
|
||||
for (const auto & token : input_tokens) {
|
||||
chunks.push_back(fake_input({ token }));
|
||||
}
|
||||
} else {
|
||||
size_t fed = 0;
|
||||
for (size_t i = 1; i <= tc.input.size(); ++i) {
|
||||
// Use UTF-8 safe truncation to avoid corrupting multi-byte characters
|
||||
size_t safe_len = utf8_truncate_safe_len(std::string_view(tc.input).substr(0, i));
|
||||
chunks.emplace_back(tc.input.substr(fed, safe_len - fed));
|
||||
fed = safe_len;
|
||||
}
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < chunks.size(); ++i) {
|
||||
auto is_partial = i + 1 < chunks.size() || tc.is_partial;
|
||||
const common_chat_msg & msg_current = is_partial ? session.feed(chunks[i]) : session.finish(chunks[i]);
|
||||
|
||||
for (const auto & diff : common_chat_msg_diff::compute_diffs(msg_prev, msg_current)) {
|
||||
if (!diff.reasoning_content_delta.empty()) {
|
||||
@@ -1222,10 +1296,15 @@ static void test_peg_parser(common_chat_templates * tmpls,
|
||||
while (msg_accum.tool_calls.size() <= diff.tool_call_index) {
|
||||
msg_accum.tool_calls.push_back({ "", "", "" });
|
||||
}
|
||||
// Always update name and id from diff (may change during incremental parsing), but only if the delta
|
||||
// actually contains them
|
||||
// The name and id are set once the delta carries them.
|
||||
// A name that is already set must not change, clients concatenate name deltas.
|
||||
if (!diff.tool_call_delta.name.empty()) {
|
||||
msg_accum.tool_calls[diff.tool_call_index].name = diff.tool_call_delta.name;
|
||||
auto & name = msg_accum.tool_calls[diff.tool_call_index].name;
|
||||
if (!name.empty() && name != diff.tool_call_delta.name) {
|
||||
throw std::runtime_error("Tool call " + std::to_string(diff.tool_call_index) + " name changed from " +
|
||||
name + " to " + diff.tool_call_delta.name + " during incremental parsing");
|
||||
}
|
||||
name = diff.tool_call_delta.name;
|
||||
}
|
||||
if (!diff.tool_call_delta.id.empty()) {
|
||||
msg_accum.tool_calls[diff.tool_call_index].id = diff.tool_call_delta.id;
|
||||
@@ -1413,9 +1492,10 @@ static void test_peg_parser(common_chat_templates * tmpls,
|
||||
// sampler was already active before model output began — constrain from the
|
||||
// start of the model output (i.e. tc.input).
|
||||
std::string constrained = full_input;
|
||||
size_t constrain_from = 0;
|
||||
bool grammar_triggered = false;
|
||||
if (earliest_trigger_pos != std::string::npos) {
|
||||
auto constrain_from = std::max(earliest_trigger_pos, gen_prompt.size());
|
||||
constrain_from = std::max(earliest_trigger_pos, gen_prompt.size());
|
||||
constrained = full_input.substr(constrain_from);
|
||||
grammar_triggered = true;
|
||||
} else if (!sampling.grammar_lazy) {
|
||||
@@ -1425,7 +1505,9 @@ static void test_peg_parser(common_chat_templates * tmpls,
|
||||
|
||||
// Test the constrained portion against the grammar
|
||||
if (grammar_triggered && !tc.is_partial) {
|
||||
auto result = match_string_detailed(constrained, grammar.get());
|
||||
auto full_tokens = fake_tokenize(specials, gen_prompt);
|
||||
full_tokens.insert(full_tokens.end(), input_tokens.begin(), input_tokens.end());
|
||||
auto result = match_string_detailed(fake_tokens_from(full_tokens, constrain_from), grammar.get());
|
||||
if (!result.success) {
|
||||
std::string error_msg;
|
||||
if (result.incomplete) {
|
||||
@@ -1514,6 +1596,7 @@ class peg_tester {
|
||||
common_chat_templates_ptr tmpls_;
|
||||
std::string template_path_;
|
||||
bool detailed_debug_;
|
||||
common_peg_tokens specials_;
|
||||
friend class peg_test_builder;
|
||||
|
||||
public:
|
||||
@@ -1529,6 +1612,18 @@ class peg_tester {
|
||||
|
||||
const std::string & template_path() const { return template_path_; }
|
||||
|
||||
// Run the tests tokenized with these pieces as special tokens, as if the vocab defined them
|
||||
peg_tester & specials(const std::vector<std::string> & pieces) {
|
||||
llama_token id = FAKE_TEXT_TOKEN + 1;
|
||||
for (const auto & piece : pieces) {
|
||||
specials_.ids.emplace(piece, id);
|
||||
specials_.tokens.emplace(id, common_peg_token{ piece, LLAMA_TOKEN_ATTR_CONTROL });
|
||||
id++;
|
||||
}
|
||||
common_chat_templates_set_specials(tmpls_.get(), specials_);
|
||||
return *this;
|
||||
}
|
||||
|
||||
peg_test_builder test(const std::string & input);
|
||||
};
|
||||
|
||||
@@ -1649,8 +1744,10 @@ class peg_test_builder {
|
||||
return;
|
||||
}
|
||||
}
|
||||
LOG_INF("\n\x1b[38;5;126m[%s]\x1b[0m\n%s\n\n", tester_.template_path().c_str(), tc_.input.c_str());
|
||||
test_peg_parser(tester_.tmpls_.get(), [this](peg_test_case & t) { t = tc_; }, tester_.detailed_debug_);
|
||||
LOG_INF("\n\x1b[38;5;126m[%s%s]\x1b[0m\n%s\n\n", tester_.template_path().c_str(),
|
||||
tester_.specials_.ids.empty() ? "" : ", tokenized", tc_.input.c_str());
|
||||
test_peg_parser(tester_.tmpls_.get(), tester_.specials_, [this](peg_test_case & t) { t = tc_; },
|
||||
tester_.detailed_debug_);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -2269,9 +2366,17 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
|
||||
"const": "42"
|
||||
})";
|
||||
|
||||
// Special tokens of the templates that use the Qwen3-Coder format.
|
||||
// Check that the vocab of a model defines them before reuse. A model can share the syntax without the tokens.
|
||||
const std::vector<std::string> qwen3_coder_specials = {
|
||||
"<|im_start|>", "<|im_end|>", "<think>", "</think>",
|
||||
"<tool_call>", "</tool_call>", "<tool_response>", "</tool_response>",
|
||||
};
|
||||
|
||||
{
|
||||
// Qwen3.5 (basically same as Nemotron, but keeping separate tests just in case)
|
||||
auto tst = peg_tester("models/templates/Qwen3.5-4B.jinja", detailed_debug);
|
||||
tst.specials(qwen3_coder_specials);
|
||||
|
||||
tst.test("I'm\nthinking\n</think>\n\nHello, world!\nWhat's up?")
|
||||
.reasoning_format(COMMON_REASONING_FORMAT_AUTO)
|
||||
@@ -2843,6 +2948,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
|
||||
{
|
||||
// NVIDIA Nemotron-3 Nano
|
||||
auto tst = peg_tester("models/templates/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.jinja", detailed_debug);
|
||||
tst.specials(qwen3_coder_specials);
|
||||
|
||||
tst.test("Hello, world!\nWhat's up?").
|
||||
enable_thinking(false).
|
||||
@@ -3138,6 +3244,8 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
|
||||
{
|
||||
// Google Gemma 4 (tool calling with Gemma4 dict format)
|
||||
auto tst = peg_tester("models/templates/google-gemma-4-31B-it.jinja");
|
||||
tst.specials({ "<|turn>", "<turn|>", "<|channel>", "<channel|>", "<|tool_call>", "<tool_call|>",
|
||||
"<|tool_response>", "<tool_response|>", "<|tool>", "<tool|>", "<|\"|>", "<|think|>" });
|
||||
|
||||
tst.test("Hello, world!").expect(simple_assist_msg("Hello, world!")).run();
|
||||
|
||||
@@ -3766,6 +3874,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
|
||||
{
|
||||
// Qwen3-Coder (tool calling with XML-style format)
|
||||
auto tst = peg_tester("models/templates/Qwen3-Coder.jinja", detailed_debug);
|
||||
tst.specials(qwen3_coder_specials);
|
||||
|
||||
tst.test("Hello, world!\nWhat's up?").expect(message_assist).expect_reconstruction().run();
|
||||
|
||||
@@ -3794,6 +3903,20 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
|
||||
.expect(message_assist_call)
|
||||
.run();
|
||||
|
||||
// Required tool choice constrains from the first token instead of gating on a trigger
|
||||
tst.test(
|
||||
"<tool_call>\n"
|
||||
"<function=special_function>\n"
|
||||
"<parameter=arg1>\n"
|
||||
"1\n"
|
||||
"</parameter>\n"
|
||||
"</function>\n"
|
||||
"</tool_call>")
|
||||
.tools({ special_function_tool })
|
||||
.tool_choice(COMMON_CHAT_TOOL_CHOICE_REQUIRED)
|
||||
.expect(message_assist_call)
|
||||
.run();
|
||||
|
||||
tst.test(
|
||||
"Let me call it.\n"
|
||||
"<function=special_function>\n"
|
||||
@@ -5895,6 +6018,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
|
||||
|
||||
{
|
||||
auto tst = peg_tester("models/templates/LFM3.jinja", detailed_debug);
|
||||
tst.specials(qwen3_coder_specials);
|
||||
|
||||
tst.test("Hello, world!\nWhat's up?")
|
||||
.enable_thinking(false)
|
||||
@@ -7157,6 +7281,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
|
||||
// Muse Glimmer format tests
|
||||
{
|
||||
auto tst = peg_tester("models/templates/muse-glimmer.jinja", detailed_debug);
|
||||
tst.specials({ "<|start|>", "<|message|>", "<|eom|>" });
|
||||
|
||||
const std::string call_markup =
|
||||
"<atem:function_calls>\n"
|
||||
@@ -7166,7 +7291,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
|
||||
"</atem:function_calls>";
|
||||
|
||||
// A plain answer is unaffected
|
||||
tst.test(" to=user<|message|>Hello, world!\nWhat's up?<|eot|>")
|
||||
tst.test(" to=user<|message|>Hello, world!\nWhat's up?")
|
||||
.reasoning_format(COMMON_REASONING_FORMAT_AUTO)
|
||||
.expect(message_assist)
|
||||
.run();
|
||||
@@ -7177,7 +7302,34 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
|
||||
.tools({ special_function_tool })
|
||||
.reasoning_format(COMMON_REASONING_FORMAT_AUTO)
|
||||
.expect(message_assist_call)
|
||||
.expect_rules({ "tool-0" })
|
||||
.expect_rules({ "tool-0", "tool-calls" })
|
||||
.run();
|
||||
|
||||
// Parallel calls, one message per call
|
||||
tst.test(" to=special_function<|message|>" + call_markup + "<|eom|>"
|
||||
"<|start|>assistant to=special_function_with_opt<|message|>"
|
||||
"<atem:function_calls>\n"
|
||||
"<atem:invoke name=\"special_function_with_opt\">\n"
|
||||
"<atem:parameter name=\"arg1\">1</atem:parameter>\n"
|
||||
"<atem:parameter name=\"arg2\">2</atem:parameter>\n"
|
||||
"</atem:invoke>\n"
|
||||
"</atem:function_calls>")
|
||||
.tools({ special_function_tool, special_function_tool_with_optional_param })
|
||||
.parallel_tool_calls(true)
|
||||
.reasoning_format(COMMON_REASONING_FORMAT_AUTO)
|
||||
.expect_tool_calls({
|
||||
{ "special_function", R"({"arg1": 1})", {} },
|
||||
{ "special_function_with_opt", R"({"arg1": 1, "arg2": 2})", {} },
|
||||
})
|
||||
.run();
|
||||
|
||||
// Required tool choice after a reasoning message
|
||||
tst.test(" to=self<|message|>I'm\nthinking<|eom|>"
|
||||
"<|start|>assistant to=special_function<|message|>" + call_markup)
|
||||
.tools({ special_function_tool })
|
||||
.tool_choice(COMMON_CHAT_TOOL_CHOICE_REQUIRED)
|
||||
.reasoning_format(COMMON_REASONING_FORMAT_AUTO)
|
||||
.expect(message_assist_call_thoughts)
|
||||
.run();
|
||||
|
||||
// "Inform then act": the model answers the user and calls a tool in ONE generation,
|
||||
@@ -7193,7 +7345,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
|
||||
|
||||
// Markup quoted in an answer has no preceding <|eom|>, so it stays content instead of
|
||||
// becoming an invocation the user never asked for
|
||||
tst.test(" to=user<|message|>You invoke it like this:\n" + call_markup + "<|eot|>")
|
||||
tst.test(" to=user<|message|>You invoke it like this:\n" + call_markup)
|
||||
.tools({ special_function_tool })
|
||||
.reasoning_format(COMMON_REASONING_FORMAT_AUTO)
|
||||
.expect_content("You invoke it like this:\n" + call_markup)
|
||||
@@ -7217,7 +7369,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
|
||||
|
||||
// Tool markup inside the analysis channel is reasoning, not a call
|
||||
tst.test(" to=self<|message|>I could use " + call_markup + " here<|eom|>"
|
||||
"<|start|>assistant to=user<|message|>Hello!<|eot|>")
|
||||
"<|start|>assistant to=user<|message|>Hello!")
|
||||
.tools({ special_function_tool })
|
||||
.reasoning_format(COMMON_REASONING_FORMAT_AUTO)
|
||||
.expect_reasoning("I could use " + call_markup + " here")
|
||||
@@ -7288,6 +7440,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
|
||||
// GPT-OSS format tests
|
||||
{
|
||||
auto tst = peg_tester("models/templates/openai-gpt-oss-120b.jinja", detailed_debug);
|
||||
tst.specials({ "<|start|>", "<|end|>", "<|message|>", "<|channel|>", "<|constrain|>", "<|return|>", "<|call|>" });
|
||||
|
||||
// Basic content only - final channel
|
||||
tst.test("<|channel|>final<|message|>Hello, world!\nWhat's up?").expect(message_assist).run();
|
||||
@@ -7327,7 +7480,30 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
|
||||
.reasoning_format(COMMON_REASONING_FORMAT_AUTO)
|
||||
.tools({ special_function_tool })
|
||||
.expect(message_assist_call)
|
||||
.expect_rules({ "tool-0" })
|
||||
.expect_rules({ "tool-0-in-role", "tool-0-in-channel" })
|
||||
.run();
|
||||
|
||||
// A tool whose name prefixes another's must not be reported while the header is still open
|
||||
tst.test(" to=functions.special_function_with_opt<|channel|>analysis<|message|>{\"arg1\": 1}")
|
||||
.reasoning_format(COMMON_REASONING_FORMAT_AUTO)
|
||||
.tools({ special_function_tool, special_function_tool_with_optional_param })
|
||||
.expect(simple_assist_msg("", "", "special_function_with_opt", "{\"arg1\": 1}"))
|
||||
.run();
|
||||
|
||||
tst.test("<|channel|>commentary to=functions.special_function_with_opt<|message|>{\"arg1\": 1}")
|
||||
.reasoning_format(COMMON_REASONING_FORMAT_AUTO)
|
||||
.tools({ special_function_tool, special_function_tool_with_optional_param })
|
||||
.expect(simple_assist_msg("", "", "special_function_with_opt", "{\"arg1\": 1}"))
|
||||
.run();
|
||||
|
||||
// Required tool choice constrains from the first token instead of gating on a trigger
|
||||
tst.test(
|
||||
"<|channel|>analysis<|message|>I'm\nthinking<|end|>"
|
||||
"<|start|>assistant to=functions.special_function<|channel|>analysis<|message|>{\"arg1\": 1}")
|
||||
.reasoning_format(COMMON_REASONING_FORMAT_AUTO)
|
||||
.tools({ special_function_tool })
|
||||
.tool_choice(COMMON_CHAT_TOOL_CHOICE_REQUIRED)
|
||||
.expect(message_assist_call_thoughts)
|
||||
.run();
|
||||
|
||||
// Tool call with recipient in channel header: "<|channel|>analysis to=functions.NAME<|message|>JSON"
|
||||
@@ -7544,6 +7720,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
|
||||
|
||||
{
|
||||
auto tst = peg_tester("models/templates/StepFun3.5-Flash.jinja", detailed_debug);
|
||||
tst.specials(qwen3_coder_specials);
|
||||
|
||||
tst.test("I was thinking\n</think>\nNow I'm not.").
|
||||
enable_thinking(true).
|
||||
|
||||
@@ -41,15 +41,21 @@ struct token_and_piece {
|
||||
std::string piece;
|
||||
};
|
||||
|
||||
// token() encodes a 32-bit ID as 5 bytes: a 0xff marker followed by the ID in big-endian order.
|
||||
static std::string token(llama_token id) {
|
||||
// token() encodes a token and its piece as a 0xff marker, the ID in big-endian order, the piece length, and the piece.
|
||||
static std::string token(llama_token id, const std::string & piece) {
|
||||
return std::string{
|
||||
static_cast<char>(0xff),
|
||||
static_cast<char>((id >> 24) & 0xff),
|
||||
static_cast<char>((id >> 16) & 0xff),
|
||||
static_cast<char>((id >> 8) & 0xff),
|
||||
static_cast<char>(id & 0xff)
|
||||
};
|
||||
static_cast<char>(id & 0xff),
|
||||
static_cast<char>(piece.size())
|
||||
} + piece;
|
||||
}
|
||||
|
||||
// token() without a piece encodes a token whose piece is its <[id]> spelling.
|
||||
static std::string token(llama_token id) {
|
||||
return token(id, "<[" + std::to_string(id) + "]>");
|
||||
}
|
||||
|
||||
// parse_tokens() parses the token encodes above and UTF-8 text.
|
||||
@@ -60,7 +66,7 @@ static std::vector<token_and_piece> parse_tokens(const std::string & input) {
|
||||
while (offset < input.size()) {
|
||||
try {
|
||||
if (static_cast<unsigned char>(input[offset]) == 0xff) {
|
||||
if (offset + 5 > input.size()) {
|
||||
if (offset + 6 > input.size()) {
|
||||
throw std::runtime_error("not enough bytes for token id");
|
||||
}
|
||||
uint32_t val =
|
||||
@@ -68,9 +74,12 @@ static std::vector<token_and_piece> parse_tokens(const std::string & input) {
|
||||
(static_cast<unsigned char>(input[offset + 2]) << 16) |
|
||||
(static_cast<unsigned char>(input[offset + 3]) << 8) |
|
||||
(static_cast<unsigned char>(input[offset + 4]));
|
||||
auto piece = "<[" + std::to_string(val) + "]>";
|
||||
result.push_back({static_cast<llama_token>(val), piece});
|
||||
offset += 5;
|
||||
size_t len = static_cast<unsigned char>(input[offset + 5]);
|
||||
if (offset + 6 + len > input.size()) {
|
||||
throw std::runtime_error("not enough bytes for token piece");
|
||||
}
|
||||
result.push_back({static_cast<llama_token>(val), input.substr(offset + 6, len)});
|
||||
offset += 6 + len;
|
||||
} else {
|
||||
uint32_t cpt = unicode_cpt_from_utf8(input, offset);
|
||||
result.push_back({0, unicode_cpt_to_utf8(cpt)});
|
||||
@@ -84,23 +93,42 @@ static std::vector<token_and_piece> parse_tokens(const std::string & input) {
|
||||
return result;
|
||||
}
|
||||
|
||||
// Returns true if the sampler mask allows the token. The mask rejects a token only when every stack rejects it.
|
||||
static bool mask_allows(llama_grammar * grammar, const token_and_piece & in) {
|
||||
const auto decoded = llama_grammar_decode_utf8(in.piece, grammar->partial_utf8);
|
||||
|
||||
llama_grammar_candidates rejects = { { 0, decoded.first.data(), decoded.second, in.token, llama_grammar_is_opaque(*grammar, in.token) } };
|
||||
for (const auto & stack : llama_grammar_get_stacks(grammar)) {
|
||||
rejects = llama_grammar_reject_candidates_for_stack(llama_grammar_get_rules(grammar), stack, rejects);
|
||||
}
|
||||
return rejects.empty();
|
||||
}
|
||||
|
||||
static bool match_string(const std::string & input, llama_grammar * grammar) {
|
||||
const auto parsed = parse_tokens(input);
|
||||
|
||||
auto & stacks_cur = llama_grammar_get_stacks(grammar);
|
||||
|
||||
for (const auto & in : parsed) {
|
||||
const bool allowed = mask_allows(grammar, in);
|
||||
|
||||
// the mask decides what can be sampled, so it has to agree with what accepting the token does
|
||||
try {
|
||||
llama_grammar_accept_token(*grammar, in.token, in.piece);
|
||||
} catch (const std::runtime_error & /*e*/) {
|
||||
// normally this shouldn't get hit because of llama_grammar_apply
|
||||
return false;
|
||||
}
|
||||
if (allowed) {
|
||||
fprintf(stderr, "❌ (token %d \"%s\" is allowed by the mask but not accepted)\n", in.token, in.piece.c_str());
|
||||
}
|
||||
assert(!allowed);
|
||||
|
||||
if (stacks_cur.empty()) {
|
||||
// no stacks means that the grammar failed to match at this point
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!allowed) {
|
||||
fprintf(stderr, "❌ (token %d \"%s\" is masked but accepted)\n", in.token, in.piece.c_str());
|
||||
}
|
||||
assert(allowed);
|
||||
}
|
||||
|
||||
for (const auto & stack : stacks_cur) {
|
||||
@@ -121,6 +149,7 @@ static void test(const std::string & test_desc, const std::string & grammar_str,
|
||||
|
||||
// Save the original grammar stacks so that we can reset after every new string we want to test
|
||||
const llama_grammar_stacks stacks_org = llama_grammar_get_stacks(grammar); // copy
|
||||
const llama_partial_utf8 partial_org = grammar->partial_utf8;
|
||||
|
||||
llama_grammar_stacks & stacks_cur = llama_grammar_get_stacks(grammar);
|
||||
|
||||
@@ -160,6 +189,7 @@ static void test(const std::string & test_desc, const std::string & grammar_str,
|
||||
|
||||
// Reset the grammar stacks
|
||||
stacks_cur = stacks_org;
|
||||
grammar->partial_utf8 = partial_org;
|
||||
}
|
||||
|
||||
fprintf(stderr, " 🟠 Invalid strings:\n");
|
||||
@@ -180,6 +210,7 @@ static void test(const std::string & test_desc, const std::string & grammar_str,
|
||||
|
||||
// Reset the grammar stacks
|
||||
stacks_cur = stacks_org;
|
||||
grammar->partial_utf8 = partial_org;
|
||||
}
|
||||
|
||||
// Clean up allocated memory
|
||||
@@ -522,6 +553,137 @@ static void test_simple_grammar() {
|
||||
);
|
||||
}
|
||||
|
||||
static void test_token_rule_utf8() {
|
||||
// A byte-level token can hold part of a multibyte character, a token rule still takes it as a whole token
|
||||
test_grammar(
|
||||
"byte-split characters under token rules",
|
||||
R"""(
|
||||
root ::= <[10]> content <[11]>
|
||||
content ::= (!<[11]>)*)""",
|
||||
// Passing strings
|
||||
{
|
||||
token(10) + token(20, "\xF0") + token(21, "\x9F\xA6\x99") + token(11),
|
||||
token(10) + token(22, "\xE2") + token(23, "\x98\x80") + token(11),
|
||||
token(10) + token(24, "a\xF0") + token(21, "\x9F\xA6\x99") + token(11),
|
||||
},
|
||||
// Failing strings
|
||||
{
|
||||
token(10) + token(20, "\xF0"),
|
||||
token(10) + token(20, "\xF0") + token(11),
|
||||
}
|
||||
);
|
||||
|
||||
// A token rule can only start where a token starts, not after char rules consumed part of it
|
||||
test_grammar(
|
||||
"token rule after a char rule",
|
||||
R"""(
|
||||
root ::= "a" <[11]>)""",
|
||||
// Passing strings
|
||||
{
|
||||
"a" + token(11),
|
||||
token(12, "a") + token(11),
|
||||
},
|
||||
// Failing strings
|
||||
{
|
||||
token(11, "ab"),
|
||||
token(12, "ab") + token(11),
|
||||
}
|
||||
);
|
||||
|
||||
// A token partly matched by a char rule stays partly matched for every alternative, whichever is checked first
|
||||
test_grammar(
|
||||
"token rule alternative after a char rule",
|
||||
R"""(
|
||||
root ::= "a" (!<[7]> | "bx"))""",
|
||||
// Passing strings
|
||||
{
|
||||
"abx",
|
||||
token(12, "abx"),
|
||||
"a" + token(5),
|
||||
},
|
||||
// Failing strings
|
||||
{
|
||||
token(5, "aby"),
|
||||
token(5, "ab"),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
static void test_token_sets() {
|
||||
test_grammar(
|
||||
"token set with alternatives and ranges",
|
||||
R"""(
|
||||
root ::= <[1,5-9,12]>+)""",
|
||||
// Passing strings
|
||||
{
|
||||
token(1),
|
||||
token(5) + token(9),
|
||||
token(7) + token(12) + token(1),
|
||||
},
|
||||
// Failing strings
|
||||
{
|
||||
token(2),
|
||||
token(4),
|
||||
token(10),
|
||||
token(13),
|
||||
token(5) + token(11),
|
||||
}
|
||||
);
|
||||
|
||||
test_grammar(
|
||||
"negated token set",
|
||||
R"""(
|
||||
root ::= <[10]> (!<[11,20-29]>)* <[11]>)""",
|
||||
// Passing strings
|
||||
{
|
||||
token(10) + token(11),
|
||||
token(10) + "text" + token(12) + token(19) + token(30) + token(11),
|
||||
},
|
||||
// Failing strings
|
||||
{
|
||||
token(10) + token(20) + token(11),
|
||||
token(10) + token(29) + token(11),
|
||||
token(10) + token(11) + token(11),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
static void test_opaque_tokens() {
|
||||
// Char rules do not match a token that a token rule names, other tokens still match as text
|
||||
test_grammar(
|
||||
"tokens named by token rules are opaque to char rules",
|
||||
R"""(
|
||||
root ::= [^x]* <[11]> "done")""",
|
||||
// Passing strings
|
||||
{
|
||||
"abc" + token(11) + "done",
|
||||
token(11) + "done",
|
||||
token(12) + token(11) + "done",
|
||||
},
|
||||
// Failing strings
|
||||
{
|
||||
"abc" + token(11) + "more" + token(11) + "done",
|
||||
token(11) + token(11) + "done",
|
||||
}
|
||||
);
|
||||
|
||||
test_grammar(
|
||||
"tokens named by a negated set are opaque to char rules",
|
||||
R"""(
|
||||
root ::= [^x]* !<[11,20-29]>)""",
|
||||
// Passing strings
|
||||
{
|
||||
"abc" + token(30),
|
||||
token(12) + token(30),
|
||||
},
|
||||
// Failing strings
|
||||
{
|
||||
token(25) + token(30),
|
||||
token(11) + token(30),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
static void test_complex_grammar() {
|
||||
// Test case for a more complex grammar, with both failure strings and success strings
|
||||
test_grammar(
|
||||
@@ -1539,6 +1701,9 @@ int main() {
|
||||
fprintf(stdout, "Running grammar integration tests...\n");
|
||||
test_simple_grammar();
|
||||
test_complex_grammar();
|
||||
test_token_rule_utf8();
|
||||
test_token_sets();
|
||||
test_opaque_tokens();
|
||||
test_special_chars();
|
||||
test_quantifiers();
|
||||
test_failure_missing_root();
|
||||
|
||||
@@ -179,6 +179,14 @@ int main()
|
||||
verify_failure(deep_grammar.c_str());
|
||||
}
|
||||
|
||||
verify_failure(R"""(
|
||||
root ::= <[9-5]>
|
||||
)""");
|
||||
|
||||
verify_failure(R"""(
|
||||
root ::= <[4294967296]>
|
||||
)""");
|
||||
|
||||
verify_parsing(R"""(
|
||||
root ::= "a"{0,5000}
|
||||
)""", {
|
||||
@@ -594,5 +602,21 @@ int main()
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
});
|
||||
|
||||
verify_parsing(R"""(
|
||||
root ::= <[1,5-9,12]> !<[11,20-29]>
|
||||
)""", {
|
||||
{"root", 0}
|
||||
}, {
|
||||
// root (index 0)
|
||||
{LLAMA_GRETYPE_TOKEN, 1},
|
||||
{LLAMA_GRETYPE_TOKEN_ALT, 5},
|
||||
{LLAMA_GRETYPE_TOKEN_RNG_UPPER, 9},
|
||||
{LLAMA_GRETYPE_TOKEN_ALT, 12},
|
||||
{LLAMA_GRETYPE_TOKEN_NOT, 11},
|
||||
{LLAMA_GRETYPE_TOKEN_ALT, 20},
|
||||
{LLAMA_GRETYPE_TOKEN_RNG_UPPER, 29},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
});
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -202,7 +202,7 @@ int main()
|
||||
uint32_t *cp = new uint32_t[2]; // dynamically allocate memory for code_point
|
||||
cp[0] = 37 + i;
|
||||
cp[1] = 0;
|
||||
next_candidates[i] = {i, cp, {}, 0};
|
||||
next_candidates[i] = {i, cp, {}, 0, false};
|
||||
}
|
||||
|
||||
std::vector<std::vector<std::pair<uint32_t, uint16_t>>> expected_reject = {
|
||||
|
||||
Reference in new issue
Block a user