From 108f73adfe607a035ffa9d5996545c669a5decdf Mon Sep 17 00:00:00 2001 From: o7si <32285332+o7si@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:52:21 +0800 Subject: [PATCH] grammar : optimize matching with left factoring (#29557) * grammar : limit the number of alternatives * grammar : introduce an AST for grammar parsing * grammar : revert AST refactor and alternatives limit This reverts commits: - 84f3b57c4d6c482e7187a30735d16b02de3784fa - 393a707ebb7e6f798abdb8af34cb1bc102f4aaa6 * grammar: optimize big grammar matching by using left-factorization * grammar : preserve character class boundaries during left factoring * grammar : prevent infinite expansion during left factoring * grammar : use ASCII characters in comments * grammar : test empty suffixes after left factoring * grammar : deduplicate left factoring logic * grammar : add comments * grammar : use number parameters and clarify RSS output * grammar : clarify character class test names * grammar : remove performance benchmark * grammar : share immutable rules --------- Co-authored-by: Piotr Wilkin --- src/llama-grammar.cpp | 222 +++++++++++++++++++++++------ src/llama-grammar.h | 5 +- tests/test-grammar-integration.cpp | 26 ++++ 3 files changed, 208 insertions(+), 45 deletions(-) diff --git a/src/llama-grammar.cpp b/src/llama-grammar.cpp index 4e096a368..0ef01dbca 100644 --- a/src/llama-grammar.cpp +++ b/src/llama-grammar.cpp @@ -948,6 +948,160 @@ static void llama_grammar_advance_stack( } } +// Left-factor grammar rules to reduce redundant parallel stacks. +// +// When a rule has many alternates (e.g. tool-choice ::= tc0 | tc1 | ... | tc119) +// and the referenced rules share a common element-content prefix, extract that +// prefix so the parser processes it once instead of N times. +// +// Handles two cases: +// 1. Inline common prefix: rule = "ab1" | "ab2" -> rule = "ab" rule', rule' = "1" | "2" +// 2. Cross-rule prefix: rule = A | B | C where A,B,C start with same elements +// -> rule = common_prefix rule', rule' = A_suffix | B_suffix | C_suffix +static void llama_grammar_left_factor(llama_grammar_rules & rules) { + auto elem_eq = [](const llama_grammar_element & a, const llama_grammar_element & b) { + return a.type == b.type && a.value == b.value; + }; + + // Keep the prefix boundary outside character classes. + auto trim_prefix = [](const llama_grammar_rule & rule, size_t start, size_t length) { + while (length > 0) { + const auto type = rule[start + length].type; + if (type != LLAMA_GRETYPE_CHAR_ALT && type != LLAMA_GRETYPE_CHAR_RNG_UPPER) { + break; + } + length--; + } + return length; + }; + + struct alt_info { + size_t rule_idx; + size_t start; + size_t end; + }; + + const size_t n_rules_original = rules.size(); + // Original rules factor at most once; generated rules do not resolve references. + // Each new suffix removes a non-empty prefix. + bool changed = true; + while (changed) { + changed = false; + const size_t n_rules = rules.size(); + + for (size_t ri = 0; ri < n_rules; ri++) { + const auto & rule = rules[ri]; + + // Split rule into alternates + std::vector> alt_ranges; + size_t alt_start = 0; + for (size_t j = 0; j < rule.size(); j++) { + if (rule[j].type == LLAMA_GRETYPE_ALT || rule[j].type == LLAMA_GRETYPE_END) { + alt_ranges.push_back({alt_start, j}); + alt_start = j + 1; + } + } + + if (alt_ranges.size() < 4) { + continue; + } + + // Resolve each alternate: if it's a single RULE_REF, follow it to get the + // actual element sequence from the referenced rule. + std::vector resolved; + resolved.reserve(alt_ranges.size()); + bool all_single_refs = true; + + for (const auto & [astart, aend] : alt_ranges) { + size_t len = aend - astart; + // Resolving references in generated suffixes can expand recursive rules forever. + if (ri < n_rules_original && len == 1 && rule[astart].type == LLAMA_GRETYPE_RULE_REF) { + size_t ref_id = static_cast(rule[astart].value); + if (ref_id < rules.size()) { + size_t ref_end = 0; + for (size_t k = 0; k < rules[ref_id].size(); k++) { + if (rules[ref_id][k].type == LLAMA_GRETYPE_ALT || + rules[ref_id][k].type == LLAMA_GRETYPE_END) { + ref_end = k; + break; + } + } + bool single_alt = (rules[ref_id][ref_end].type == LLAMA_GRETYPE_END); + if (single_alt) { + resolved.push_back({ref_id, 0, ref_end}); + continue; + } + } + } + all_single_refs = false; + resolved.push_back({ri, astart, aend}); + } + + if (!all_single_refs) { + resolved.clear(); + for (const auto & [astart, aend] : alt_ranges) { + resolved.push_back({ri, astart, aend}); + } + } + + size_t prefix_len = 0; + bool prefix_match = true; + while (prefix_match) { + const auto & first = resolved[0]; + if (first.start + prefix_len >= first.end) { + break; + } + const auto & ref_elem = rules[first.rule_idx][first.start + prefix_len]; + + for (size_t ai = 1; ai < resolved.size(); ai++) { + const auto & alt = resolved[ai]; + if (alt.start + prefix_len >= alt.end || + !elem_eq(rules[alt.rule_idx][alt.start + prefix_len], ref_elem)) { + prefix_match = false; + break; + } + } + if (prefix_match) { + prefix_len++; + } + } + + for (const auto & alt : resolved) { + prefix_len = trim_prefix(rules[alt.rule_idx], alt.start, prefix_len); + } + if (prefix_len == 0) { + continue; + } + + uint32_t suffix_choice_id = static_cast(rules.size()); + llama_grammar_rule suffix_choice; + + for (size_t ai = 0; ai < resolved.size(); ai++) { + if (ai > 0) { + suffix_choice.push_back({LLAMA_GRETYPE_ALT, 0}); + } + const auto & alt = resolved[ai]; + for (size_t j = alt.start + prefix_len; j < alt.end; j++) { + suffix_choice.push_back(rules[alt.rule_idx][j]); + } + } + suffix_choice.push_back({LLAMA_GRETYPE_END, 0}); + + llama_grammar_rule new_rule; + const auto & first = resolved[0]; + for (size_t j = 0; j < prefix_len; j++) { + new_rule.push_back(rules[first.rule_idx][first.start + j]); + } + new_rule.push_back({LLAMA_GRETYPE_RULE_REF, suffix_choice_id}); + new_rule.push_back({LLAMA_GRETYPE_END, 0}); + + rules.push_back(std::move(suffix_choice)); + rules[ri] = std::move(new_rule); + changed = true; + } + } +} + static bool llama_grammar_detect_left_recursion( const llama_grammar_rules & rules, size_t rule_index, @@ -1006,7 +1160,7 @@ static bool llama_grammar_detect_left_recursion( } const llama_grammar_rules & llama_grammar_get_rules(const struct llama_grammar * grammar) { - return grammar->rules; + return *grammar->rules; } llama_grammar_stacks & llama_grammar_get_stacks(struct llama_grammar * grammar) { @@ -1035,7 +1189,7 @@ static void llama_grammar_accept_chr( if (!llama_grammar_is_end_of_sequence(match.second)) { new_stack.push_back(match.second); } - llama_grammar_advance_stack(grammar.rules, new_stack, new_stacks); + llama_grammar_advance_stack(*grammar.rules, new_stack, new_stacks); } } @@ -1221,12 +1375,16 @@ struct llama_grammar * llama_grammar_init_impl( } } - // Check for left recursion - std::vector rules_visited(n_rules); - std::vector rules_in_progress(n_rules); - std::vector rules_may_be_empty(n_rules); + // Left-factor rules to reduce redundant parallel stacks + llama_grammar_left_factor(vec_rules); + + // Check for left recursion (use post-factoring rule count) + const size_t n_rules_total = vec_rules.size(); + std::vector rules_visited(n_rules_total); + std::vector rules_in_progress(n_rules_total); + std::vector rules_may_be_empty(n_rules_total); try { - for (size_t i = 0; i < n_rules; i++) { + for (size_t i = 0; i < n_rules_total; i++) { if (rules_visited[i]) { continue; } @@ -1267,7 +1425,7 @@ struct llama_grammar * llama_grammar_init_impl( // then the pointers would be invalidated when the local vec_rules goes out of scope. return new llama_grammar { vocab, - std::move(vec_rules), + std::make_shared(std::move(vec_rules)), std::move(stacks), /* .partial_utf8 = */ {}, /* .lazy = */ false, @@ -1319,12 +1477,16 @@ struct llama_grammar * llama_grammar_init_impl( vec_rules[i].push_back({LLAMA_GRETYPE_END, 0}); } - // Check for left recursion - std::vector rules_visited(n_rules); - std::vector rules_in_progress(n_rules); - std::vector rules_may_be_empty(n_rules); + // Left-factor rules to reduce redundant parallel stacks + llama_grammar_left_factor(vec_rules); + + // Check for left recursion (use post-factoring rule count) + const size_t n_rules_total = vec_rules.size(); + std::vector rules_visited(n_rules_total); + std::vector rules_in_progress(n_rules_total); + std::vector rules_may_be_empty(n_rules_total); try { - for (size_t i = 0; i < n_rules; i++) { + for (size_t i = 0; i < n_rules_total; i++) { if (rules_visited[i]) { continue; } @@ -1378,7 +1540,7 @@ struct llama_grammar * llama_grammar_init_impl( // then the pointers would be invalidated when the local vec_rules goes out of scope. return new llama_grammar { vocab, - std::move(vec_rules), + std::make_shared(std::move(vec_rules)), std::move(stacks), /* .partial_utf8 = */ {}, /* .lazy = */ lazy, @@ -1399,33 +1561,7 @@ void llama_grammar_free_impl(struct llama_grammar * grammar) { } struct llama_grammar * llama_grammar_clone_impl(const struct llama_grammar & grammar) { - auto * result = new llama_grammar { - grammar.vocab, - grammar.rules, - grammar.stacks, - grammar.partial_utf8, - grammar.lazy, - grammar.awaiting_trigger, - grammar.trigger_buffer, - grammar.trigger_buffer_positions, - grammar.trigger_tokens, - grammar.trigger_patterns, - }; - - // redirect elements in stacks to point to new rules - for (size_t is = 0; is < result->stacks.size(); is++) { - for (size_t ie = 0; ie < result->stacks[is].size(); ie++) { - for (size_t ir0 = 0; ir0 < grammar.rules.size(); ir0++) { - for (size_t ir1 = 0; ir1 < grammar.rules[ir0].size(); ir1++) { - if (grammar.stacks[is][ie] == &grammar.rules[ir0][ir1]) { - result->stacks[is][ie] = &result->rules[ir0][ir1]; - } - } - } - } - } - - return result; + return new llama_grammar(grammar); } void llama_grammar_apply_impl(const struct llama_grammar & grammar, llama_token_data_array * cur_p) { @@ -1465,7 +1601,7 @@ void llama_grammar_apply_impl(const struct llama_grammar & grammar, llama_token_ } } - const auto rejects = llama_grammar_reject_candidates(grammar.rules, grammar.stacks, candidates_grammar); + const auto rejects = llama_grammar_reject_candidates(*grammar.rules, grammar.stacks, candidates_grammar); for (const auto & reject : rejects) { cur_p->data[reject.index].logit = -INFINITY; } @@ -1566,7 +1702,7 @@ void llama_grammar_accept_token(struct llama_grammar & grammar, llama_token toke if (!llama_grammar_is_end_of_sequence(pos + 1)) { new_stack.push_back(pos + 1); } - llama_grammar_advance_stack(grammar.rules, new_stack, stacks_new); + llama_grammar_advance_stack(*grammar.rules, new_stack, stacks_new); } } else { llama_grammar_stacks current_stacks = {stack}; diff --git a/src/llama-grammar.h b/src/llama-grammar.h index 2885f7705..5829d8edc 100644 --- a/src/llama-grammar.h +++ b/src/llama-grammar.h @@ -3,6 +3,7 @@ #include "llama.h" #include +#include #include #include #include @@ -132,8 +133,8 @@ struct llama_grammar { // note: allow null vocab for testing (not great) const llama_vocab * vocab; - const llama_grammar_rules rules; // TODO: shared ptr - llama_grammar_stacks stacks; + const std::shared_ptr rules; + llama_grammar_stacks stacks; // buffer for partially generated UTF-8 sequence from accepted tokens llama_partial_utf8 partial_utf8; diff --git a/tests/test-grammar-integration.cpp b/tests/test-grammar-integration.cpp index 75bfca645..a7a7e968e 100644 --- a/tests/test-grammar-integration.cpp +++ b/tests/test-grammar-integration.cpp @@ -193,6 +193,19 @@ static void test_schema(const std::string & test_desc, const std::string & schem } static void test_simple_grammar() { + test_grammar( + "left factor leading empty suffix", + "root ::= \"ab\" | \"abc\" | \"abd\" | \"abe\"", + {"ab", "abc", "abd", "abe"}, {"", "a", "abb", "abcc"}); + test_grammar( + "left factor middle empty suffix", + "root ::= \"abc\" | \"ab\" | \"abd\" | \"abe\"", + {"ab", "abc", "abd", "abe"}, {"", "a", "abb", "abcc"}); + test_grammar( + "left factor recursive references", + "root ::= \"ok\" | choice\nchoice ::= a | b | c | d\na ::= \"x\" a\nb ::= \"x\" b\nc ::= \"x\" c\nd ::= \"x\" d", + {"ok"}, {"", "x", "xx", "okx"}); + test_schema( "min 0", R"""({ @@ -623,6 +636,19 @@ static void test_special_chars() { "šŸ”µšŸŸ abcšŸŸ šŸ”µ" } ); + + test_grammar( + "skip left factoring inside character classes", + "root ::= [ab] \"x\" | [ac] \"x\" | [ad] \"x\" | [ae] \"x\"", + {"ax", "bx", "cx", "dx", "ex"}, {"", "a", "fx", "axx"}); + test_grammar( + "skip left factoring inside character ranges", + "root ::= [a-b] \"x\" | [a-c] \"x\" | [a-d] \"x\" | [a-e] \"x\"", + {"ax", "bx", "cx", "dx", "ex"}, {"", "a", "fx", "axx"}); + test_grammar( + "left factor referenced character alternatives", + "root ::= a | b | c | d\na ::= \"p\" [a] \"x\"\nb ::= \"p\" [ab] \"x\"\nc ::= \"p\" [ac] \"x\"\nd ::= \"p\" [ad] \"x\"", + {"pax", "pbx", "pcx", "pdx"}, {"", "px", "pex", "paxx"}); } static void test_quantifiers() {