Files
llama.cpp/common/peg-parser.cpp
T

2263 lines
90 KiB
C++

#include "peg-parser.h"
#include "common.h"
#include "json-schema-to-grammar.h"
#include "log.h"
#include "trie.h"
#include "unicode.h"
#include <algorithm>
#include <initializer_list>
#include <map>
#include <memory>
#include <set>
#include <stdexcept>
// The parser can run with a lower limit than COMMON_RECURSIVE_LIMIT
static constexpr int RULE_RECURSIVE_LIMIT = 64;
// Trick to catch missing branches
template <typename T>
inline constexpr bool is_always_false_v = false;
const char * common_peg_parse_result_type_name(common_peg_parse_result_type type) {
switch (type) {
case COMMON_PEG_PARSE_RESULT_FAIL: return "fail";
case COMMON_PEG_PARSE_RESULT_SUCCESS: return "success";
case COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT: return "need_more_input";
default: return "unknown";
}
}
static bool is_hex_digit(const char c) {
return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
}
static std::pair<uint32_t, size_t> parse_hex_escape(const std::string & str, size_t pos, int hex_count) {
if (pos + hex_count > str.length()) {
return {0, 0};
}
uint32_t value = 0;
for (int i = 0; i < hex_count; i++) {
char c = str[pos + i];
if (!is_hex_digit(c)) {
return {0, 0};
}
value <<= 4;
if ('a' <= c && c <= 'f') {
value += c - 'a' + 10;
} else if ('A' <= c && c <= 'F') {
value += c - 'A' + 10;
} else if ('0' <= c && c <= '9') {
value += c - '0';
} else {
break;
}
}
return {value, static_cast<size_t>(hex_count)};
}
static std::pair<uint32_t, size_t> parse_char_class_char(const std::string & content, size_t pos) {
if (content[pos] == '\\' && pos + 1 < content.length()) {
switch (content[pos + 1]) {
case 'x': {
auto result = parse_hex_escape(content, pos + 2, 2);
if (result.second > 0) {
return {result.first, 2 + result.second};
}
// Invalid escape, treat as literal 'x'
return {static_cast<uint32_t>('x'), 2};
}
case 'u': {
auto result = parse_hex_escape(content, pos + 2, 4);
if (result.second > 0) {
return {result.first, 2 + result.second};
}
// Invalid escape, treat as literal 'u'
return {static_cast<uint32_t>('u'), 2};
}
case 'U': {
auto result = parse_hex_escape(content, pos + 2, 8);
if (result.second > 0) {
return {result.first, 2 + result.second};
}
// Invalid escape, treat as literal 'U'
return {static_cast<uint32_t>('U'), 2};
}
case 'n': return {'\n', 2};
case 't': return {'\t', 2};
case 'r': return {'\r', 2};
case '\\': return {'\\', 2};
case ']': return {']', 2};
case '[': return {'[', 2};
default: return {static_cast<uint32_t>(content[pos + 1]), 2};
}
}
// Regular character - return as codepoint
return {static_cast<uint32_t>(static_cast<unsigned char>(content[pos])), 1};
}
static std::pair<std::vector<common_peg_chars_parser::char_range>, bool> parse_char_classes(const std::string & classes) {
std::vector<common_peg_chars_parser::char_range> ranges;
bool negated = false;
std::string content = classes;
if (content.front() == '[') {
content = content.substr(1);
}
if (content.back() == ']') {
content.pop_back();
}
// Check for negation
if (!content.empty() && content.front() == '^') {
negated = true;
content = content.substr(1);
}
size_t i = 0;
while (i < content.length()) {
auto [start, start_len] = parse_char_class_char(content, i);
i += start_len;
if (i + 1 < content.length() && content[i] == '-') {
// Range detected
auto [end, end_len] = parse_char_class_char(content, i + 1);
ranges.push_back(common_peg_chars_parser::char_range{start, end});
i += 1 + end_len;
} else {
ranges.push_back(common_peg_chars_parser::char_range{start, start});
}
}
return {ranges, negated};
}
common_peg_ast_id common_peg_ast_arena::find_by_tag(const common_peg_ast_node & parent, const std::string & tag, int max_depth) const {
for (auto child_id : parent.children) {
const auto & child = get(child_id);
if (child.tag == tag) {
return child_id;
}
if (max_depth > 1) {
auto result = find_by_tag(child, tag, max_depth - 1);
if (result != COMMON_PEG_INVALID_AST_ID) {
return result;
}
}
}
return COMMON_PEG_INVALID_AST_ID;
}
common_peg_ast_id common_peg_ast_arena::find_by_rule(const common_peg_ast_node & parent, const std::string & rule, int max_depth) const {
for (auto child_id : parent.children) {
const auto & child = get(child_id);
if (child.rule == rule) {
return child_id;
}
if (max_depth > 1) {
auto result = find_by_rule(child, rule, max_depth - 1);
if (result != COMMON_PEG_INVALID_AST_ID) {
return result;
}
}
}
return COMMON_PEG_INVALID_AST_ID;
}
std::string common_peg_ast_node::sanitized_text() const {
if (invalid_utf8.empty()) {
return std::string(text);
}
std::string out;
out.reserve(text.size() + 2 * invalid_utf8.size());
size_t seg_start = start;
for (const auto & invalid : invalid_utf8) {
out.append(text.data() + (seg_start - start), invalid.pos - seg_start);
out.append("\xEF\xBF\xBD");
seg_start = invalid.pos + invalid.len;
}
out.append(text.data() + (seg_start - start), end - seg_start);
return out;
}
void common_peg_ast_arena::visit(common_peg_ast_id id, const common_peg_ast_visitor & visitor) const {
// no recursion: the AST is several nodes deep per level of the input
std::vector<common_peg_ast_id> stack = { id };
while (!stack.empty()) {
const auto cur = stack.back();
stack.pop_back();
if (cur == COMMON_PEG_INVALID_AST_ID) {
continue;
}
const auto & node = get(cur);
visitor(node);
stack.insert(stack.end(), node.children.rbegin(), node.children.rend());
}
}
void common_peg_ast_arena::visit(const common_peg_parse_result & result, const common_peg_ast_visitor & visitor) const {
for (const auto & node : result.nodes) {
visit(node, visitor);
}
}
common_peg_tokens::common_peg_tokens(const llama_vocab * vocab) {
const llama_token n_tokens = llama_vocab_n_tokens(vocab);
for (llama_token id = 0; id < n_tokens; ++id) {
const auto attr = llama_vocab_get_attr(vocab, id);
if (attr & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED)) {
std::string text = llama_vocab_get_text(vocab, id);
ids.emplace(text, id);
tokens.emplace(id, common_peg_token{ std::move(text), attr });
}
}
}
llama_token common_peg_tokens::token_id(const std::string & text) const {
auto it = ids.find(text);
return it != ids.end() ? it->second : LLAMA_TOKEN_NULL;
}
// Insert every delimiter the parser matches into the trie, starting from each node in from. Returns the end nodes.
// A choice adds the delimiters of each branch. A sequence joins every combination of its children.
static std::set<size_t> insert_delimiter(common_trie & trie, const common_peg_arena & arena, common_peg_parser_id id,
std::set<size_t> from = { 0 }) {
return std::visit([&](const auto & p) -> std::set<size_t> {
using T = std::decay_t<decltype(p)>;
if constexpr (std::is_same_v<T, common_peg_tag_parser> || std::is_same_v<T, common_peg_atomic_parser>) {
return insert_delimiter(trie, arena, p.child, std::move(from));
} else if constexpr (std::is_same_v<T, common_peg_literal_parser>) {
std::set<size_t> ends;
for (size_t node : from) {
size_t pos = 0;
while (pos < p.literal.size()) {
auto result = common_parse_utf8_codepoint(p.literal, pos);
if (result.status != utf8_parse_result::SUCCESS) {
break;
}
node = trie.extend(node, common_trie::symbol::codepoint(result.codepoint));
pos += result.bytes_consumed;
}
ends.insert(node);
}
return ends;
} else if constexpr (std::is_same_v<T, common_peg_token_parser>) {
std::set<size_t> ends;
for (size_t node : from) {
ends.insert(trie.extend(node, common_trie::symbol::token(p.token)));
}
return ends;
} else if constexpr (std::is_same_v<T, common_peg_choice_parser>) {
std::set<size_t> ends;
for (auto child : p.children) {
ends.merge(insert_delimiter(trie, arena, child, from));
}
return ends;
} else if constexpr (std::is_same_v<T, common_peg_sequence_parser>) {
for (auto child : p.children) {
from = insert_delimiter(trie, arena, child, std::move(from));
}
return from;
} else {
throw std::invalid_argument("a delimiter may only be a sequence of literals, tokens, and choices of them, possibly wrapped in tags and atomics");
}
}, arena.get(id));
}
// Build the matcher of an until parser, each pattern valued by the branch it belongs to
static common_trie build_until_matcher(const common_peg_arena & arena,
const std::vector<common_peg_until_branch> & branches,
bool include) {
common_trie matcher;
for (size_t b = 0; b < branches.size(); b++) {
for (size_t end : insert_delimiter(matcher, arena, branches[b].delimiter)) {
if (end == 0) {
throw std::invalid_argument("delimiter must not match the empty string");
}
// nothing follows a delimiter without include, so the first branch to claim it wins
auto owner = matcher.mark(end, (int32_t) b);
if (owner != (int32_t) b && include) {
throw std::invalid_argument("until branches " + std::to_string(owner) + " and " + std::to_string(b) + " must not share a delimiter");
}
}
}
return matcher;
}
struct parser_executor;
common_peg_parser_id common_peg_arena::add_parser(common_peg_parser_variant parser) {
common_peg_parser_id id = parsers_.size();
parsers_.push_back(std::move(parser));
return id;
}
void common_peg_arena::add_rule(const std::string & name, common_peg_parser_id id) {
rules_[name] = id;
}
common_peg_parser_id common_peg_arena::get_rule(const std::string & name) const {
auto it = rules_.find(name);
if (it == rules_.end()) {
throw std::runtime_error("Rule not found: " + name);
}
return it->second;
}
struct parser_executor {
const common_peg_arena & arena;
common_peg_parse_context & ctx;
size_t start_pos;
parser_executor(const common_peg_arena & arena, common_peg_parse_context & ctx, size_t start)
: arena(arena), ctx(ctx), start_pos(start) {}
std::string debug_indent() const { return std::string(ctx.parse_depth * 2, ' '); }
std::string debug_input_snippet(size_t pos, size_t len = 60) const {
if (pos >= ctx.input.size()) {
return "<EOF>";
}
auto snippet = ctx.input.substr(pos, len);
// Escape newlines for display
std::string result;
for (char c : snippet) {
if (c == '\n') {
result += "\\n";
} else if (c == '\r') {
result += "\\r";
} else if (c == '\t') {
result += "\\t";
} else {
result += c;
}
}
if (pos + len < ctx.input.size()) {
result += "...";
}
return result;
}
common_peg_parse_result operator()(const common_peg_epsilon_parser & /* p */) const {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos);
}
common_peg_parse_result operator()(const common_peg_start_parser & /* p */) const {
return common_peg_parse_result(
start_pos == 0 ? COMMON_PEG_PARSE_RESULT_SUCCESS : COMMON_PEG_PARSE_RESULT_FAIL,
start_pos
);
}
common_peg_parse_result operator()(const common_peg_end_parser & /* p */) const {
return common_peg_parse_result(
start_pos >= ctx.input.size() ? COMMON_PEG_PARSE_RESULT_SUCCESS : COMMON_PEG_PARSE_RESULT_FAIL,
start_pos
);
}
common_peg_parse_result operator()(const common_peg_literal_parser & p) {
auto pos = start_pos;
for (auto i = 0u; i < p.literal.size(); ++i) {
if (pos >= ctx.input.size()) {
if (!ctx.is_lenient()) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos);
}
if (ctx.input[pos] != p.literal[i]) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
}
++pos;
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos);
}
common_peg_parse_result operator()(const common_peg_token_parser & p) {
if (start_pos >= ctx.input.size()) {
if (!ctx.is_lenient()) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos);
}
if (start_pos >= ctx.tokens.size() || ctx.tokens[start_pos] != p.token) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
}
// Skip the rest of the piece, up to the next token or the end of the input
auto pos = start_pos + 1;
while (pos < ctx.input.size() && ctx.tokens[pos] == LLAMA_TOKEN_NULL) {
++pos;
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos);
}
common_peg_parse_result operator()(const common_peg_token_not_parser & p) {
if (start_pos >= ctx.input.size()) {
if (!ctx.is_lenient()) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos);
}
if (start_pos >= ctx.tokens.size() || ctx.tokens[start_pos] == LLAMA_TOKEN_NULL ||
std::find(p.tokens.begin(), p.tokens.end(), ctx.tokens[start_pos]) != p.tokens.end()) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
}
// Skip the rest of the piece, up to the next token or the end of the input
auto pos = start_pos + 1;
while (pos < ctx.input.size() && ctx.tokens[pos] == LLAMA_TOKEN_NULL) {
++pos;
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos);
}
common_peg_parse_result operator()(const common_peg_sequence_parser & p) {
if (ctx.is_debug()) {
LOG_DBG("%sSEQ start at %zu '%s' (%zu children)\n", debug_indent().c_str(), start_pos,
debug_input_snippet(start_pos).c_str(), p.children.size());
}
ctx.parse_depth++;
auto pos = start_pos;
std::vector<common_peg_ast_id> nodes;
std::vector<common_peg_invalid_utf8> invalid_utf8;
for (size_t i = 0; i < p.children.size(); i++) {
const auto & child_id = p.children[i];
if (ctx.is_debug()) {
fprintf(stderr, "%sSEQ child %zu: %s\n", debug_indent().c_str(), i, arena.dump(child_id).c_str());
}
auto result = arena.parse(child_id, ctx, pos);
if (ctx.is_debug()) {
fprintf(stderr, "%sSEQ child %zu: %s at %zu->%zu\n", debug_indent().c_str(), i,
common_peg_parse_result_type_name(result.type), result.start, result.end);
}
if (result.fail()) {
ctx.parse_depth--;
if (ctx.is_debug()) {
fprintf(stderr, "%sSEQ -> FAIL\n", debug_indent().c_str());
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos, result.end);
}
if (!result.nodes.empty()) {
nodes.insert(nodes.end(), result.nodes.begin(), result.nodes.end());
}
invalid_utf8.insert(invalid_utf8.end(), result.invalid_utf8.begin(), result.invalid_utf8.end());
if (result.need_more_input()) {
ctx.parse_depth--;
if (ctx.is_debug()) {
fprintf(stderr, "%sSEQ -> NEED_MORE\n", debug_indent().c_str());
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, result.end, std::move(nodes), std::move(invalid_utf8));
}
pos = result.end;
}
ctx.parse_depth--;
if (ctx.is_debug()) {
fprintf(stderr, "%sSEQ -> SUCCESS at %zu->%zu\n", debug_indent().c_str(), start_pos, pos);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, std::move(nodes), std::move(invalid_utf8));
}
common_peg_parse_result operator()(const common_peg_choice_parser & p) {
if (ctx.is_debug()) {
fprintf(stderr, "%sCHOICE start at %zu '%s' (%zu options)\n", debug_indent().c_str(), start_pos,
debug_input_snippet(start_pos).c_str(), p.children.size());
}
ctx.parse_depth++;
auto pos = start_pos;
for (size_t i = 0; i < p.children.size(); i++) {
const auto & child_id = p.children[i];
if (ctx.is_debug()) {
fprintf(stderr, "%sCHOICE option %zu: %s\n", debug_indent().c_str(), i, arena.dump(child_id).c_str());
}
auto result = arena.parse(child_id, ctx, pos);
if (ctx.is_debug()) {
fprintf(stderr, "%sCHOICE option %zu: %s\n", debug_indent().c_str(), i,
common_peg_parse_result_type_name(result.type));
}
if (!result.fail()) {
ctx.parse_depth--;
if (ctx.is_debug()) {
fprintf(stderr, "%sCHOICE -> %s (option %zu)\n", debug_indent().c_str(),
common_peg_parse_result_type_name(result.type), i);
}
return result;
}
}
ctx.parse_depth--;
if (ctx.is_debug()) {
fprintf(stderr, "%sCHOICE -> FAIL (no options matched)\n", debug_indent().c_str());
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
}
common_peg_parse_result operator()(const common_peg_repetition_parser & p) {
if (ctx.is_debug()) {
fprintf(stderr, "%sREPEAT start at %zu '%s' (min=%d, max=%d)\n", debug_indent().c_str(), start_pos,
debug_input_snippet(start_pos).c_str(), p.min_count, p.max_count);
}
ctx.parse_depth++;
auto pos = start_pos;
int match_count = 0;
std::vector<common_peg_ast_id> nodes;
std::vector<common_peg_invalid_utf8> invalid_utf8;
// Try to match up to max_count times (or unlimited if max_count is -1)
while (p.max_count == -1 || match_count < p.max_count) {
if (pos >= ctx.input.size()) {
if (ctx.is_debug()) {
fprintf(stderr, "%sREPEAT: at end of input, count=%d\n", debug_indent().c_str(), match_count);
}
break;
}
auto result = arena.parse(p.child, ctx, pos);
if (ctx.is_debug()) {
fprintf(stderr, "%sREPEAT iter %d: %s at %zu->%zu, nodes=%zu\n", debug_indent().c_str(), match_count,
common_peg_parse_result_type_name(result.type), result.start, result.end, result.nodes.size());
fprintf(stderr, "%sREPEAT CHILD: %s\n", debug_indent().c_str(), arena.dump(p.child).c_str());
}
if (result.success()) {
// Prevent infinite loop on empty matches
if (result.end == pos) {
if (ctx.is_debug()) {
fprintf(stderr, "%s REPEAT: empty match, stopping\n", debug_indent().c_str());
}
break;
}
if (!result.nodes.empty()) {
nodes.insert(nodes.end(), result.nodes.begin(), result.nodes.end());
}
invalid_utf8.insert(invalid_utf8.end(), result.invalid_utf8.begin(), result.invalid_utf8.end());
pos = result.end;
match_count++;
continue;
}
if (result.need_more_input()) {
if (!result.nodes.empty()) {
nodes.insert(nodes.end(), result.nodes.begin(), result.nodes.end());
}
invalid_utf8.insert(invalid_utf8.end(), result.invalid_utf8.begin(), result.invalid_utf8.end());
ctx.parse_depth--;
if (ctx.is_debug()) {
fprintf(stderr, "%sREPEAT -> NEED_MORE (count=%d, nodes=%zu)\n", debug_indent().c_str(),
match_count, nodes.size());
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, result.end, std::move(nodes), std::move(invalid_utf8));
}
// Child failed - stop trying
if (ctx.is_debug()) {
fprintf(stderr, "%sREPEAT: child failed, stopping\n", debug_indent().c_str());
}
break;
}
// Check if we got enough matches
if (p.min_count > 0 && match_count < p.min_count) {
ctx.parse_depth--;
if (pos >= ctx.input.size() && ctx.is_lenient()) {
if (ctx.is_debug()) {
fprintf(stderr, "%sREPEAT -> NEED_MORE (not enough matches: %d < %d)\n", debug_indent().c_str(),
match_count, p.min_count);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos, std::move(nodes), std::move(invalid_utf8));
}
if (ctx.is_debug()) {
fprintf(stderr, "%sREPEAT -> FAIL (not enough matches: %d < %d)\n", debug_indent().c_str(), match_count,
p.min_count);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos, pos);
}
ctx.parse_depth--;
if (ctx.is_debug()) {
fprintf(stderr, "%sREPEAT -> SUCCESS (count=%d, nodes=%zu)\n", debug_indent().c_str(), match_count,
nodes.size());
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, std::move(nodes), std::move(invalid_utf8));
}
common_peg_parse_result operator()(const common_peg_and_parser & p) {
auto result = arena.parse(p.child, ctx, start_pos);
// Pass result but don't consume input
return common_peg_parse_result(result.type, start_pos);
}
common_peg_parse_result operator()(const common_peg_not_parser & p) {
auto result = arena.parse(p.child, ctx, start_pos);
if (result.success()) {
// Fail if the underlying parser matches
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
}
if (result.need_more_input()) {
// Propagate - need to know what child would match before negating
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos);
}
// Child failed, so negation succeeds
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos);
}
common_peg_parse_result operator()(const common_peg_any_parser & /* p */) const {
// Parse a single UTF-8 codepoint (not just a single byte)
auto result = common_parse_utf8_codepoint(ctx.input, start_pos);
if (result.status == utf8_parse_result::INCOMPLETE) {
if (!ctx.is_lenient()) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos);
}
if (result.status == utf8_parse_result::INVALID) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, start_pos + result.bytes_consumed);
}
common_peg_parse_result operator()(const common_peg_space_parser & /* p */) {
auto pos = start_pos;
while (pos < ctx.input.size()) {
auto c = static_cast<unsigned char>(ctx.input[pos]);
if (std::isspace(c)) {
++pos;
} else {
break;
}
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos);
}
common_peg_parse_result operator()(const common_peg_chars_parser & p) const {
auto pos = start_pos;
int match_count = 0;
// Try to match up to max_count times (or unlimited if max_count is -1)
while (p.max_count == -1 || match_count < p.max_count) {
auto result = common_parse_utf8_codepoint(ctx.input, pos);
if (result.status == utf8_parse_result::INCOMPLETE) {
if (match_count >= p.min_count) {
// We have enough matches, succeed with what we have
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos);
}
// Not enough matches yet
if (!ctx.is_lenient()) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos);
}
if (result.status == utf8_parse_result::INVALID) {
// Malformed UTF-8 in input
if (match_count >= p.min_count) {
// We have enough matches, succeed up to here
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos);
}
// Not enough matches, fail
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
}
// Check if this codepoint matches our character class
bool matches = false;
for (const auto & range : p.ranges) {
if (range.contains(result.codepoint)) {
matches = true;
break;
}
}
// If negated, invert the match result
if (p.negated) {
matches = !matches;
}
if (matches) {
pos += result.bytes_consumed;
++match_count;
} else {
// Character doesn't match, stop matching
break;
}
}
// Check if we got enough matches
if (match_count < p.min_count) {
if (pos >= ctx.input.size() && ctx.is_lenient()) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos, pos);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos);
}
static common_peg_parse_result handle_escape_sequence(common_peg_parse_context & ctx, size_t start, size_t & pos, const char delimiter) {
auto save = pos;
++pos; // consume '\'
if (pos >= ctx.input.size()) {
if (!ctx.is_lenient()) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start);
}
pos = save; // suppress unmatched '\'
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start, pos);
}
char c = ctx.input[pos];
if (c == delimiter || c == '\\' || c == '/' || c == 'b' || c == 'f' || c == 'n' || c == 'r' || c == 't') {
++pos;
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start, pos);
}
if (c == 'u') {
auto result = handle_unicode_escape(ctx, start, pos);
if (result.need_more_input()) {
pos = save; // suppress incomplete sequence
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start, pos);
}
return result;
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start);
}
static common_peg_parse_result handle_unicode_escape(common_peg_parse_context & ctx, size_t start, size_t & pos) {
++pos; // consume 'u'
for (int i = 0; i < 4; ++i) {
if (pos >= ctx.input.size()) {
if (!ctx.is_lenient()) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start, pos);
}
if (!is_hex_digit(ctx.input[pos])) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start);
}
++pos;
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start, pos);
}
common_peg_parse_result operator()(const common_peg_string_parser & p) {
auto pos = start_pos;
// Parse string content (without quotes)
while (pos < ctx.input.size()) {
char c = ctx.input[pos];
if (c == p.delimiter) {
// Found closing delimiter - success (don't consume it)
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos);
}
if (c == '\\') {
auto result = handle_escape_sequence(ctx, start_pos, pos, p.delimiter);
if (!result.success()) {
return result;
}
} else {
auto utf8_result = common_parse_utf8_codepoint(ctx.input, pos);
if (utf8_result.status == utf8_parse_result::INCOMPLETE) {
if (!ctx.is_lenient()) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos);
}
if (utf8_result.status == utf8_parse_result::INVALID) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
}
pos += utf8_result.bytes_consumed;
}
}
// Reached end without finding closing quote
if (!ctx.is_lenient()) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos, pos);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos);
}
common_peg_parse_result operator()(const common_peg_until_parser & p) const {
if (p.include) {
return parse_through(p);
}
const auto & matcher = p.matcher;
// Scan input and check for delimiters
size_t pos = start_pos;
size_t last_valid_pos = start_pos;
std::vector<common_peg_invalid_utf8> invalid_utf8;
while (pos < ctx.input.size()) {
auto step = matcher.next(ctx.input, ctx.tokens, pos);
if (step.status == utf8_parse_result::INCOMPLETE && ctx.is_lenient()) {
// The rest of the sequence may still arrive, return what we have so far
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, last_valid_pos, {}, std::move(invalid_utf8));
}
if (step.status != utf8_parse_result::SUCCESS) {
// Malformed UTF-8, or a sequence truncated by the end of a complete input.
// A delimiter cannot start inside bytes that fail to decode, so consume them and move on
invalid_utf8.push_back({pos, step.bytes_consumed});
pos += step.bytes_consumed;
last_valid_pos = pos;
continue;
}
// Check if a delimiter starts at this position
auto match = matcher.check_at(ctx.input, ctx.tokens, pos);
if (match == common_trie::COMPLETE_MATCH) {
// Found a complete delimiter, return everything before it
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, {}, std::move(invalid_utf8));
}
if (match == common_trie::PARTIAL_MATCH && !ctx.is_complete()) {
// Found a partial match extending to end of input, return everything before it.
// On a complete input the delimiter can no longer form, so the bytes belong to the text
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, {}, std::move(invalid_utf8));
}
pos += step.bytes_consumed;
last_valid_pos = pos;
}
if (last_valid_pos == ctx.input.size() && ctx.is_lenient()) {
// Reached the end of a partial stream, there might still be more input that we need to consume.
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, last_valid_pos, {}, std::move(invalid_utf8));
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, last_valid_pos, {}, std::move(invalid_utf8));
}
// Parse content, then the first branch whose delimiter and rest follow it, like C (D1 R1 | D2 R2 | ...)
common_peg_parse_result parse_through(const common_peg_until_parser & p) const {
auto content = arena.parse(p.content, ctx, start_pos);
if (content.fail() || content.need_more_input()) {
return content;
}
auto pos = content.end;
for (const auto & branch : p.branches) {
auto nodes = content.nodes;
auto invalid_utf8 = content.invalid_utf8;
auto end = pos;
bool matched = true;
for (auto id : { branch.delimiter, branch.rest }) {
if (id == COMMON_PEG_INVALID_PARSER_ID) {
continue;
}
auto result = arena.parse(id, ctx, end);
if (result.fail()) {
matched = false;
break;
}
nodes.insert(nodes.end(), result.nodes.begin(), result.nodes.end());
invalid_utf8.insert(invalid_utf8.end(), result.invalid_utf8.begin(), result.invalid_utf8.end());
end = result.end;
if (result.need_more_input()) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, end, std::move(nodes), std::move(invalid_utf8));
}
}
if (matched) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, end, std::move(nodes), std::move(invalid_utf8));
}
}
if (p.optional && pos == ctx.input.size()) {
auto type = ctx.is_lenient() ? COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT : COMMON_PEG_PARSE_RESULT_SUCCESS;
return common_peg_parse_result(type, start_pos, pos, std::move(content.nodes), std::move(content.invalid_utf8));
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos, pos);
}
common_peg_parse_result operator()(const common_peg_schema_parser & p) {
return arena.parse(p.child, ctx, start_pos);
}
common_peg_parse_result operator()(const common_peg_rule_parser & p) {
// only rules recurse, a recursive rule is entered once per input level
if (ctx.rule_depth[p.child] >= RULE_RECURSIVE_LIMIT) {
throw common_peg_recursion_error("input is nested more than " + std::to_string(RULE_RECURSIVE_LIMIT) + " levels deep");
}
// Parse the child
ctx.rule_depth[p.child]++;
auto result = arena.parse(p.child, ctx, start_pos);
ctx.rule_depth[p.child]--;
if (!result.fail()) {
std::string_view text;
if (result.start < ctx.input.size()) {
text = std::string_view(ctx.input).substr(result.start, result.end - result.start);
}
auto node_id = ctx.ast.add_node(
p.name,
"",
result.start,
result.end,
text,
std::move(result.nodes),
result.need_more_input(),
result.invalid_utf8
);
return common_peg_parse_result(result.type, result.start, result.end, { node_id }, std::move(result.invalid_utf8));
}
return result;
}
common_peg_parse_result operator()(const common_peg_tag_parser & p) {
// Parse the child
if (ctx.is_debug()) {
fprintf(stderr, "%sTAG: %s\n", debug_indent().c_str(), p.tag.c_str());
}
auto result = arena.parse(p.child, ctx, start_pos);
if (!result.fail()) {
std::string_view text;
if (result.start < ctx.input.size()) {
text = std::string_view(ctx.input).substr(result.start, result.end - result.start);
}
auto node_id = ctx.ast.add_node(
"",
p.tag,
result.start,
result.end,
text,
std::move(result.nodes),
result.need_more_input(),
result.invalid_utf8
);
return common_peg_parse_result(result.type, result.start, result.end, { node_id }, std::move(result.invalid_utf8));
}
return result;
}
common_peg_parse_result operator()(const common_peg_ref_parser & p) {
auto rule_id = arena.get_rule(p.name);
return arena.parse(rule_id, ctx, start_pos);
}
common_peg_parse_result operator()(const common_peg_atomic_parser & p) {
auto result = arena.parse(p.child, ctx, start_pos);
if (result.need_more_input()) {
// Clear nodes so they don't propagate up.
result.nodes.clear();
}
return result;
}
common_peg_parse_result operator()(const common_peg_gbnf_parser & p) {
return arena.parse(p.child, ctx, start_pos);
}
};
common_peg_parse_result common_peg_arena::parse(common_peg_parse_context & ctx, size_t start) const {
if (root_ == COMMON_PEG_INVALID_PARSER_ID) {
throw std::runtime_error("No root parser set");
}
// Initialize rule depth for all parsers
ctx.rule_depth.assign(parsers_.size(), 0);
return parse(root_, ctx, start);
}
common_peg_parse_result common_peg_arena::parse(common_peg_parser_id id, common_peg_parse_context & ctx, size_t start) const {
// Execute parser
const auto & parser = parsers_.at(id);
parser_executor exec(*this, ctx, start);
return std::visit(exec, parser);
}
common_peg_parser_id common_peg_arena::resolve_ref(common_peg_parser_id id) {
const auto & parser = parsers_.at(id);
if (auto ref = std::get_if<common_peg_ref_parser>(&parser)) {
return get_rule(ref->name);
}
return id;
}
static void bfs_node(common_peg_ast_arena &arena, std::ostringstream & oss, const common_peg_ast_node & node, int indent) {
for (int i = 0; i < indent; i++) {
oss << " ";
}
oss << "NODE " << node.id;
if (!node.rule.empty()) {
oss << " (rule " << node.rule << ")";
}
if (!node.tag.empty()) {
oss << " (tag " << node.tag << ")";
}
oss << " ['" << node.text << "']\n";
for (const auto child : node.children) {
bfs_node(arena, oss, arena.get(child), indent + 1);
}
}
std::string common_peg_ast_arena::dump() {
std::ostringstream oss;
for (auto & node : nodes_) {
bfs_node(*this, oss, node, 0);
}
return oss.str();
}
void common_peg_arena::resolve_refs() {
// Walk through all parsers and replace refs with their corresponding rule IDs
for (auto & parser : parsers_) {
std::visit([this](auto & p) {
using T = std::decay_t<decltype(p)>;
if constexpr (std::is_same_v<T, common_peg_sequence_parser>) {
for (auto & child : p.children) {
child = resolve_ref(child);
}
} else if constexpr (std::is_same_v<T, common_peg_choice_parser>) {
for (auto & child : p.children) {
child = resolve_ref(child);
}
} else if constexpr (std::is_same_v<T, common_peg_repetition_parser> ||
std::is_same_v<T, common_peg_and_parser> ||
std::is_same_v<T, common_peg_not_parser> ||
std::is_same_v<T, common_peg_tag_parser> ||
std::is_same_v<T, common_peg_atomic_parser> ||
std::is_same_v<T, common_peg_gbnf_parser>) {
p.child = resolve_ref(p.child);
} else if constexpr (std::is_same_v<T, common_peg_until_parser>) {
if (p.content != COMMON_PEG_INVALID_PARSER_ID) {
p.content = resolve_ref(p.content);
}
for (auto & branch : p.branches) {
branch.delimiter = resolve_ref(branch.delimiter);
if (branch.rest != COMMON_PEG_INVALID_PARSER_ID) {
branch.rest = resolve_ref(branch.rest);
}
}
} else if constexpr (std::is_same_v<T, common_peg_rule_parser>) {
p.child = resolve_ref(p.child);
if (p.start != COMMON_PEG_INVALID_PARSER_ID) {
p.start = resolve_ref(p.start);
p.rest = resolve_ref(p.rest);
}
} else if constexpr (std::is_same_v<T, common_peg_schema_parser>) {
p.child = resolve_ref(p.child);
} else if constexpr (std::is_same_v<T, common_peg_epsilon_parser> ||
std::is_same_v<T, common_peg_start_parser> ||
std::is_same_v<T, common_peg_end_parser> ||
std::is_same_v<T, common_peg_ref_parser> ||
std::is_same_v<T, common_peg_literal_parser> ||
std::is_same_v<T, common_peg_token_parser> ||
std::is_same_v<T, common_peg_token_not_parser> ||
std::is_same_v<T, common_peg_string_parser> ||
std::is_same_v<T, common_peg_chars_parser> ||
std::is_same_v<T, common_peg_any_parser> ||
std::is_same_v<T, common_peg_space_parser>) {
// These rules do not have children
} else {
static_assert(is_always_false_v<T>);
}
}, parser);
}
// Also flatten root if it's a ref
if (root_ != COMMON_PEG_INVALID_PARSER_ID) {
root_ = resolve_ref(root_);
}
}
std::string common_peg_arena::dump(common_peg_parser_id id) const {
std::set<common_peg_parser_id> visited;
return dump_impl(id, visited);
}
std::string common_peg_arena::dump_impl(common_peg_parser_id id,
std::set<common_peg_parser_id> & visited) const {
// Check for cycles
if (visited.count(id)) {
return "[cycle]";
}
visited.insert(id);
const auto & parser = parsers_.at(id);
return std::visit([this, &visited](const auto & p) -> std::string {
using T = std::decay_t<decltype(p)>;
if constexpr (std::is_same_v<T, common_peg_epsilon_parser>) {
return "Epsilon";
} else if constexpr (std::is_same_v<T, common_peg_start_parser>) {
return "Start";
} else if constexpr (std::is_same_v<T, common_peg_end_parser>) {
return "End";
} else if constexpr (std::is_same_v<T, common_peg_literal_parser>) {
return "Literal(" + p.literal + ")";
} else if constexpr (std::is_same_v<T, common_peg_token_parser>) {
return "Token(" + p.piece + ", " + std::to_string(p.token) + ")";
} else if constexpr (std::is_same_v<T, common_peg_token_not_parser>) {
std::vector<std::string> parts;
for (auto token : p.tokens) {
parts.push_back(std::to_string(token));
}
return "TokenNot(" + string_join(parts, ", ") + ")";
} else if constexpr (std::is_same_v<T, common_peg_sequence_parser>) {
std::vector<std::string> parts;
for (const auto & child : p.children) {
parts.push_back(dump_impl(child, visited));
}
return "Sequence(" + string_join(parts, ", ") + ")";
} else if constexpr (std::is_same_v<T, common_peg_choice_parser>) {
std::vector<std::string> parts;
for (const auto & child : p.children) {
parts.push_back(dump_impl(child, visited));
}
return "Choice(" + string_join(parts, ", ") + ")";
} else if constexpr (std::is_same_v<T, common_peg_repetition_parser>) {
if (p.max_count == -1) {
return "Repetition(" + dump_impl(p.child, visited) + ", " + std::to_string(p.min_count) +
", unbounded)";
}
return "Repetition(" + dump_impl(p.child, visited) + ", " + std::to_string(p.min_count) + ", " + std::to_string(p.max_count) + ")";
} else if constexpr (std::is_same_v<T, common_peg_and_parser>) {
return "And(" + dump_impl(p.child, visited) + ")";
} else if constexpr (std::is_same_v<T, common_peg_not_parser>) {
return "Not(" + dump_impl(p.child, visited) + ")";
} else if constexpr (std::is_same_v<T, common_peg_atomic_parser>) {
return "Atomic(" + dump_impl(p.child, visited) + ")";
} else if constexpr (std::is_same_v<T, common_peg_gbnf_parser>) {
return "Gbnf(" + p.grammar + ", " + dump_impl(p.child, visited) + ")";
} else if constexpr (std::is_same_v<T, common_peg_any_parser>) {
return "Any";
} else if constexpr (std::is_same_v<T, common_peg_space_parser>) {
return "Space";
} else if constexpr (std::is_same_v<T, common_peg_chars_parser>) {
if (p.max_count == -1) {
return "CharRepeat(" + p.pattern + ", " + std::to_string(p.min_count) + ", unbounded)";
}
return "CharRepeat(" + p.pattern + ", " + std::to_string(p.min_count) + ", " + std::to_string(p.max_count) + ")";
} else if constexpr (std::is_same_v<T, common_peg_string_parser>) {
return "String(" + std::string(1, p.delimiter) + ")";
} else if constexpr (std::is_same_v<T, common_peg_until_parser>) {
std::vector<std::string> branches;
for (const auto & branch : p.branches) {
auto text = dump_impl(branch.delimiter, visited);
if (branch.rest != COMMON_PEG_INVALID_PARSER_ID) {
text += " " + dump_impl(branch.rest, visited);
}
branches.push_back(std::move(text));
}
if (p.include) {
return "Until(" + dump_impl(p.content, visited) + ", " + string_join(branches, " | ") + (p.optional ? ", optional" : "") + ")";
}
return "Until(" + string_join(branches, " | ") + ")";
} else if constexpr (std::is_same_v<T, common_peg_schema_parser>) {
return "Schema(" + dump_impl(p.child, visited) + ", " + (p.node ? common_chat_schema::kind_name(p.node->kind()) : "null") + ")";
} else if constexpr (std::is_same_v<T, common_peg_rule_parser>) {
return "Rule(" + p.name + ", " + dump_impl(p.child, visited) + ")";
} else if constexpr (std::is_same_v<T, common_peg_ref_parser>) {
return "Ref(" + p.name + ")";
} else if constexpr (std::is_same_v<T, common_peg_tag_parser>) {
return "Tag(" + p.tag + ", " + dump_impl(p.child, visited) + ")";
} else if constexpr (std::is_same_v<T, common_peg_atomic_parser>) {
return "Atomic(" + dump_impl(p.child, visited) + ")";
} else {
return "Unknown";
}
}, parser);
}
common_peg_parser & common_peg_parser::operator=(const common_peg_parser & other) {
id_ = other.id_;
return *this;
}
common_peg_parser & common_peg_parser::operator+=(const common_peg_parser & other) {
id_ = builder_.sequence({id_, other.id_});
return *this;
}
common_peg_parser & common_peg_parser::operator|=(const common_peg_parser & other) {
id_ = builder_.choice({id_, other.id_});
return *this;
}
common_peg_parser common_peg_parser::operator+(const common_peg_parser & other) const {
return builder_.sequence({id_, other.id_});
}
common_peg_parser common_peg_parser::operator|(const common_peg_parser & other) const {
return builder_.choice({id_, other.id_});
}
common_peg_parser common_peg_parser::operator<<(const common_peg_parser & other) const {
return builder_.sequence({id_, builder_.space(), other.id_});
}
common_peg_parser common_peg_parser::operator+(const char * str) const {
return *this + builder_.literal(str);
}
common_peg_parser common_peg_parser::operator+(const std::string & str) const {
return *this + builder_.literal(str);
}
common_peg_parser common_peg_parser::operator<<(const char * str) const {
return *this << builder_.literal(str);
}
common_peg_parser common_peg_parser::operator<<(const std::string & str) const {
return *this << builder_.literal(str);
}
common_peg_parser common_peg_parser::operator|(const char * str) const {
return *this | builder_.literal(str);
}
common_peg_parser common_peg_parser::operator|(const std::string & str) const {
return *this | builder_.literal(str);
}
common_peg_parser operator+(const char * str, const common_peg_parser & p) {
return p.builder().literal(str) + p;
}
common_peg_parser operator+(const std::string & str, const common_peg_parser & p) {
return operator+(str.c_str(), p);
}
common_peg_parser operator<<(const char * str, const common_peg_parser & p) {
return p.builder().literal(str) << p;
}
common_peg_parser operator<<(const std::string & str, const common_peg_parser & p) {
return operator<<(str.c_str(), p);
}
common_peg_parser operator|(const char * str, const common_peg_parser & p) {
return p.builder().literal(str) | p;
}
common_peg_parser operator|(const std::string & str, const common_peg_parser & p) {
return operator|(str.c_str(), p);
}
common_peg_parser_builder::common_peg_parser_builder() {}
common_peg_parser_builder::common_peg_parser_builder(common_peg_tokens tokens) {
arena_.tokens_ = std::move(tokens);
}
common_peg_parser common_peg_parser_builder::until_one_of(const std::vector<std::string> & delimiters) {
common_peg_until_parser p;
for (const auto & d : delimiters) {
p.branches.push_back({ literal(d).id() });
}
p.matcher = build_until_matcher(arena_, p.branches, p.include);
return add(std::move(p));
}
common_peg_parser common_peg_parser_builder::until(const common_peg_parser & delimiter) {
common_peg_until_parser p;
p.branches = { { delimiter.id() } };
p.matcher = build_until_matcher(arena_, p.branches, p.include);
return add(std::move(p));
}
common_peg_parser common_peg_parser_builder::through(const common_peg_parser & content, const std::vector<common_peg_until_branch> & branches, bool optional) {
if (branches.empty()) {
throw std::runtime_error("through requires at least one branch");
}
common_peg_until_parser p;
p.branches = branches;
p.include = true;
p.optional = optional;
p.content = content.id();
p.matcher = build_until_matcher(arena_, p.branches, p.include);
return add(std::move(p));
}
common_peg_parser common_peg_parser_builder::token(const std::string & piece) {
auto token = arena_.tokens_.token_id(piece);
if (token == LLAMA_TOKEN_NULL) {
return literal(piece);
}
return add(common_peg_token_parser{token, piece});
}
common_peg_parser common_peg_parser_builder::token(const std::vector<std::string> & pieces, bool negate) {
if (!negate) {
std::vector<common_peg_parser> alts;
for (const auto & piece : pieces) {
alts.push_back(token(piece));
}
return choice(alts);
}
common_peg_token_not_parser p;
for (const auto & piece : pieces) {
auto token = arena_.tokens_.token_id(piece);
if (token == LLAMA_TOKEN_NULL) {
throw std::invalid_argument("token is not registered with the builder: " + piece);
}
p.tokens.push_back(token);
}
return add(p);
}
common_peg_parser common_peg_parser_builder::sequence(const std::vector<common_peg_parser_id> & parsers) {
// Flatten nested sequences
std::vector<common_peg_parser_id> flattened;
for (const auto & p : parsers) {
const auto & parser = arena_.get(p);
if (auto seq = std::get_if<common_peg_sequence_parser>(&parser)) {
flattened.insert(flattened.end(), seq->children.begin(), seq->children.end());
} else {
flattened.push_back(p);
}
}
return wrap(arena_.add_parser(common_peg_sequence_parser{flattened}));
}
common_peg_parser common_peg_parser_builder::sequence(const std::vector<common_peg_parser> & parsers) {
std::vector<common_peg_parser_id> ids;
ids.reserve(parsers.size());
for (const auto & p : parsers) {
ids.push_back(p.id());
}
return sequence(ids);
}
common_peg_parser common_peg_parser_builder::sequence(std::initializer_list<common_peg_parser> parsers) {
std::vector<common_peg_parser_id> ids;
ids.reserve(parsers.size());
for (const auto & p : parsers) {
ids.push_back(p.id());
}
return sequence(ids);
}
common_peg_parser common_peg_parser_builder::choice(const std::vector<common_peg_parser_id> & parsers) {
// Flatten nested choices
std::vector<common_peg_parser_id> flattened;
for (const auto & p : parsers) {
const auto & parser = arena_.get(p);
if (auto choice = std::get_if<common_peg_choice_parser>(&parser)) {
flattened.insert(flattened.end(), choice->children.begin(), choice->children.end());
} else {
flattened.push_back(p);
}
}
return wrap(arena_.add_parser(common_peg_choice_parser{flattened}));
}
common_peg_parser common_peg_parser_builder::choice(const std::vector<common_peg_parser> & parsers) {
std::vector<common_peg_parser_id> ids;
ids.reserve(parsers.size());
for (const auto & p : parsers) {
ids.push_back(p.id());
}
return choice(ids);
}
common_peg_parser common_peg_parser_builder::choice(std::initializer_list<common_peg_parser> parsers) {
std::vector<common_peg_parser_id> ids;
ids.reserve(parsers.size());
for (const auto & p : parsers) {
ids.push_back(p.id());
}
return choice(ids);
}
common_peg_parser common_peg_parser_builder::chars(const std::string & classes, int min, int max) {
auto [ranges, negated] = parse_char_classes(classes);
return wrap(arena_.add_parser(common_peg_chars_parser{classes, ranges, negated, min, max}));
}
common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, common_chat_schema_document_ptr doc, const common_chat_schema & node, bool raw) {
return wrap(arena_.add_parser(common_peg_schema_parser{p.id(), name, std::move(doc), &node, raw}));
}
common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, const common_json & schema, bool raw) {
auto doc = std::make_shared<const common_chat_schema_document>(common_chat_schema_from_json(schema));
return this->schema(p, name, doc, *doc->root, raw);
}
common_peg_parser common_peg_parser_builder::rule(const std::string & name, const common_peg_parser & p, bool trigger) {
auto clean_name = gbnf_format_rule_name(name);
auto rule_id = arena_.add_parser(common_peg_rule_parser{clean_name, p.id(), trigger});
arena_.add_rule(clean_name, rule_id);
return ref(clean_name);
}
common_peg_parser common_peg_parser_builder::rule(const std::string & name, const std::function<common_peg_parser()> & builder_fn, bool trigger) {
auto clean_name = gbnf_format_rule_name(name);
if (arena_.has_rule(clean_name)) {
return ref(clean_name);
}
// Create placeholder rule to allow recursive references
auto placeholder = any(); // Temporary placeholder
auto placeholder_rule_id = arena_.add_parser(common_peg_rule_parser{clean_name, placeholder.id(), trigger});
arena_.add_rule(clean_name, placeholder_rule_id);
// Build the actual parser
auto parser = builder_fn();
// Replace placeholder with actual rule
auto rule_id = arena_.add_parser(common_peg_rule_parser{clean_name, parser.id(), trigger});
arena_.rules_[clean_name] = rule_id;
return ref(clean_name);
}
common_peg_parser common_peg_parser_builder::trigger_rule(const std::string & name, const common_peg_parser & start, const common_peg_parser & rest) {
common_trie scratch;
if (insert_delimiter(scratch, arena_, start.id()).count(0)) {
throw std::invalid_argument("trigger start must not match the empty string");
}
auto clean_name = gbnf_format_rule_name(name);
auto rule_id = arena_.add_parser(common_peg_rule_parser{clean_name, (start + rest).id(), true, start.id(), rest.id()});
arena_.add_rule(clean_name, rule_id);
return ref(clean_name);
}
void common_peg_parser_builder::set_root(const common_peg_parser & p) {
arena_.set_root(p.id());
}
common_peg_arena common_peg_parser_builder::build() {
arena_.resolve_refs();
return std::move(arena_);
}
// String primitives
common_peg_parser common_peg_parser_builder::string_content(char delimiter) {
return wrap(arena_.add_parser(common_peg_string_parser{delimiter}));
}
common_peg_parser common_peg_parser_builder::double_quoted_string() {
return rule("double-quoted-string", [this]() {
return sequence({literal("\""), string_content('"'), literal("\"")});
});
}
common_peg_parser common_peg_parser_builder::single_quoted_string() {
return rule("single-quoted-string", [this]() {
return sequence({literal("'"), string_content('\''), literal("'")});
});
}
common_peg_parser common_peg_parser_builder::quoted_string() {
return rule("quoted-string", [this]() {
return choice({double_quoted_string(), single_quoted_string()});
});
}
// JSON parsers
common_peg_parser common_peg_parser_builder::json_number() {
return rule("json-number", [this]() {
auto digit1_9 = chars("[1-9]", 1, 1);
auto digits = chars("[0-9]");
auto int_part = choice({literal("0"), sequence({digit1_9, chars("[0-9]", 0, -1)})});
auto frac = sequence({literal("."), digits});
auto exp = sequence({choice({literal("e"), literal("E")}), optional(chars("[+-]", 1, 1)), digits});
// Negative lookahead: only commit the number when the next character can't extend it.
// At EOF in partial mode, chars returns NEED_MORE → negate propagates NEED_MORE → number not committed.
// This prevents premature commits of partial numbers (e.g. "3" when "3.14" is incoming).
auto not_number_continuation = negate(chars("[0-9.eE+-]", 1, 1));
return sequence({ optional(literal("-")), int_part, optional(frac), optional(exp), not_number_continuation });
});
}
common_peg_parser common_peg_parser_builder::json_string() {
return rule("json-string", [this]() {
return sequence({literal("\""), string_content('"'), literal("\"")});
});
}
common_peg_parser common_peg_parser_builder::json_bool() {
return rule("json-bool", [this]() {
return choice({literal("true"), literal("false")});
});
}
common_peg_parser common_peg_parser_builder::json_null() {
return rule("json-null", [this]() {
return literal("null");
});
}
common_peg_parser common_peg_parser_builder::json_object() {
return rule("json-object", [this]() {
auto ws = space();
auto member = sequence({json_string(), ws, literal(":"), ws, json()});
auto members = sequence({member, zero_or_more(sequence({ws, literal(","), ws, member}))});
return sequence({
literal("{"),
ws,
choice({
literal("}"),
sequence({members, ws, literal("}")})
})
});
});
}
common_peg_parser common_peg_parser_builder::json_array() {
return rule("json-array", [this]() {
auto ws = space();
auto elements = sequence({json(), zero_or_more(sequence({ws, literal(","), ws, json()}))});
return sequence({
literal("["),
ws,
choice({
literal("]"),
sequence({elements, ws, literal("]")})
})
});
});
}
common_peg_parser common_peg_parser_builder::json() {
return rule("json-value", [this]() {
return choice({
json_object(),
json_array(),
json_string(),
json_number(),
json_bool(),
json_null()
});
});
}
common_peg_parser common_peg_parser_builder::python_string() {
return rule("python-string", [this]() {
return choice({double_quoted_string(), single_quoted_string()});
});
}
common_peg_parser common_peg_parser_builder::python_number() {
return json_number();
}
common_peg_parser common_peg_parser_builder::python_bool() {
return rule("python-bool", [this]() {
return choice({literal("True"), literal("False")});
});
}
common_peg_parser common_peg_parser_builder::python_null() {
return rule("python-none", [this]() {
return literal("None");
});
}
common_peg_parser common_peg_parser_builder::python_dict() {
return rule("python-dict", [this]() {
auto ws = space();
auto member = sequence({python_string(), ws, literal(":"), ws, python_value()});
auto members = sequence({member, zero_or_more(sequence({ws, literal(","), ws, member}))});
return sequence({
literal("{"),
ws,
choice({
literal("}"),
sequence({members, ws, literal("}")})
}),
ws
});
});
}
common_peg_parser common_peg_parser_builder::python_array() {
return rule("python-array", [this]() {
auto ws = space();
auto elements = sequence({python_value(), zero_or_more(sequence({literal(","), ws, python_value()}))});
return sequence({
literal("["),
ws,
choice({
literal("]"),
sequence({elements, ws, literal("]")})
}),
ws
});
});
}
common_peg_parser common_peg_parser_builder::python_value() {
return rule("python-value", [this]() {
return choice({
python_dict(),
python_array(),
python_string(),
python_number(),
python_bool(),
python_null()
});
});
}
common_peg_parser common_peg_parser_builder::marker() {
auto sharp_bracket_parser = literal("<") + until(">") + literal(">");
auto square_bracket_parser = literal("[") + until("]") + literal("]");
return choice({ sharp_bracket_parser, square_bracket_parser });
}
common_peg_parser common_peg_parser_builder::json_member(const std::string & key, const common_peg_parser & p) {
auto ws = space();
return sequence({
literal("\"" + key + "\""),
ws,
literal(":"),
ws,
p,
});
}
static std::string gbnf_escape_char_class(uint32_t c) {
if (c == '-' || c == ']' || c == '[' || c == '\\') {
return "\\" + std::string(1, (char) c);
}
// Escape whitespace control characters
if (c == '\n') {
return "\\n";
}
if (c == '\t') {
return "\\t";
}
if (c == '\r') {
return "\\r";
}
// Printable ASCII
if (c >= 0x20 && c <= 0x7E) {
return std::string(1, (char) c);
}
// Hex escape
char buf[16];
const char * hex = "0123456789ABCDEF";
if (c <= 0xFF) {
buf[0] = '\\';
buf[1] = 'x';
buf[2] = hex[(c >> 4) & 0xF];
buf[3] = hex[c & 0xF];
buf[4] = '\0';
} else if (c <= 0xFFFF) {
buf[0] = '\\';
buf[1] = 'u';
buf[2] = hex[(c >> 12) & 0xF];
buf[3] = hex[(c >> 8) & 0xF];
buf[4] = hex[(c >> 4) & 0xF];
buf[5] = hex[c & 0xF];
buf[6] = '\0';
} else {
buf[0] = '\\';
buf[1] = 'U';
for (int i = 0; i < 8; i++) {
buf[2 + i] = hex[(c >> ((7 - i) * 4)) & 0xF];
}
buf[10] = '\0';
}
return std::string(buf);
}
static std::string gbnf_char_class(const std::vector<uint32_t> & chars, bool negate) {
std::string s = negate ? "[^" : "[";
for (uint32_t ch : chars) {
s += gbnf_escape_char_class(ch);
}
return s + "]";
}
// Token set such as <[1,5-9]>, with consecutive ids collapsed into ranges
static std::string gbnf_token_set(const std::vector<uint32_t> & tokens, bool negate) {
std::string s = negate ? "!<[" : "<[";
for (size_t i = 0; i < tokens.size();) {
size_t j = i;
while (j + 1 < tokens.size() && tokens[j + 1] == tokens[j] + 1) {
j++;
}
s += (i == 0 ? "" : ",") + std::to_string(tokens[i]);
if (j > i) {
s += "-" + std::to_string(tokens[j]);
}
i = j + 1;
}
return s + "]>";
}
// Symbols on the transitions out of an automaton state, codepoints and tokens kept apart
struct gbnf_ac_symbols {
std::vector<uint32_t> chars;
std::vector<uint32_t> tokens;
void add(const common_trie::symbol & sym) {
(sym.is_token() ? tokens : chars).push_back(sym.value);
}
// One alternative for the codepoints and one for the tokens, each followed by next if given
std::vector<std::string> alternatives(const std::string & next = "") const {
std::vector<std::string> alts;
const std::string tail = next.empty() ? "" : " " + next;
if (!chars.empty()) {
alts.push_back(gbnf_char_class(chars, false) + tail);
}
if (!tokens.empty()) {
alts.push_back(gbnf_token_set(tokens, false) + tail);
}
return alts;
}
};
static std::string gbnf_ac_grammar(
const common_grammar_builder & builder,
const std::string & prefix,
const common_trie & trie,
const std::set<uint32_t> & opaque,
const std::function<std::string(const gbnf_ac_symbols &,
const std::map<size_t, gbnf_ac_symbols> &,
const std::map<size_t, gbnf_ac_symbols> &,
const std::string &,
const std::function<std::string(size_t)> &)> & build_rule) {
common_aho_corasick ac(trie);
// With only tokens in the alphabet, the grammar can step a whole token at a time
bool token_only = true;
for (const auto & sym : ac.alphabet) {
token_only = token_only && sym.is_token();
}
auto state_name = [&](size_t s) -> std::string {
if (s == 0) {
return prefix;
}
std::string num = std::to_string(s);
num = num.size() == 1 ? ("0" + num) : num;
return prefix + "-" + num;
};
for (size_t q = 0; q < ac.num_states(); q++) {
if (ac.is_terminal(q)) {
continue; // match states
}
std::map<size_t, gbnf_ac_symbols> buckets;
std::map<size_t, gbnf_ac_symbols> completing_to; // completing symbols by the match state they reach
gbnf_ac_symbols completing; // symbols that complete a delimiter
gbnf_ac_symbols specific; // symbols with an explicit transition
for (const auto & sym : ac.alphabet) {
size_t d = ac.next(q, sym);
if (ac.is_terminal(d)) {
completing.add(sym);
completing_to[d].add(sym);
specific.add(sym);
} else if (d != 0) {
buckets[d].add(sym); // specific non-root destination
specific.add(sym);
}
}
// every other symbol goes back to the start
std::string fallback;
if (token_only) {
fallback = gbnf_token_set(specific.tokens, true);
} else {
fallback = specific.chars.empty() ? "." : gbnf_char_class(specific.chars, true);
// Char rules never match a token named by a token rule, so allow the other named tokens by name
std::vector<uint32_t> others;
for (auto token : opaque) {
if (std::find(specific.tokens.begin(), specific.tokens.end(), token) == specific.tokens.end()) {
others.push_back(token);
}
}
if (!others.empty()) {
fallback = "(" + fallback + " | " + gbnf_token_set(others, false) + ")";
}
}
fallback += " " + state_name(0);
builder.add_rule(state_name(q), build_rule(completing, completing_to, buckets, fallback, state_name));
}
return state_name(0);
}
// GBNF grammar matching strings that contain no string in `strings` as a
// substring. Emits the complement of an Aho-Corasick automaton DFA and returns
// the start state rule name.
//
// ref: https://github.com/ggml-org/llama.cpp/pull/24839
static std::string gbnf_excluding_grammar(const common_grammar_builder & builder,
const std::string & prefix,
const common_trie & trie,
const std::set<uint32_t> & opaque) {
return gbnf_ac_grammar(builder, prefix, trie, opaque,
[](const gbnf_ac_symbols & /*completing*/,
const std::map<size_t, gbnf_ac_symbols> & /*completing_to*/,
const std::map<size_t, gbnf_ac_symbols> & buckets,
const std::string & fallback,
const std::function<std::string(size_t)> & state_name) {
// every state accepts and completing symbols get no alternative, so a forbidden string never matches
std::string rhs = "|";
for (const auto & [d, symbols] : buckets) {
for (const auto & alt : symbols.alternatives(state_name(d))) {
rhs += " " + alt + " |";
}
}
rhs += " " + fallback;
return rhs;
});
}
// GBNF grammar matching up to the first pattern in the trie, then the rest rule of that pattern's value.
// For the pattern "ab" with rest "x":
//
// prefix ::= [a] prefix-01 | [^a] prefix
// prefix-01 ::= [b] rest | [a] prefix-01 | [^ab] prefix
// rest ::= "x"
static std::string gbnf_including_grammar(const common_grammar_builder & builder,
const std::string & prefix,
const common_trie & trie,
const std::set<uint32_t> & opaque,
const std::vector<std::string> & rests,
bool optional) {
common_aho_corasick ac(trie);
// every pattern ending at a match state, the longest and the ones reached by suffix links
auto rests_at = [&](size_t s, bool & terminal) {
std::vector<std::string> alts;
terminal = false;
for (; s != 0; s = ac.fail[s]) {
auto pattern = ac.t.nodes[s].pattern;
if (pattern < 0) {
continue;
}
if (rests[pattern].empty()) {
terminal = true;
} else if (std::find(alts.begin(), alts.end(), rests[pattern]) == alts.end()) {
alts.push_back(rests[pattern]);
}
}
return alts;
};
return gbnf_ac_grammar(builder, prefix, trie, opaque,
[&](const gbnf_ac_symbols & /*completing*/,
const std::map<size_t, gbnf_ac_symbols> & completing_to,
const std::map<size_t, gbnf_ac_symbols> & buckets,
const std::string & fallback,
const std::function<std::string(size_t)> & state_name) {
std::vector<std::string> alts;
gbnf_ac_symbols terminating; // symbols that complete a pattern with no rest
for (const auto & [d, symbols] : completing_to) {
bool terminal = false;
auto rest = rests_at(d, terminal);
if (!rest.empty()) {
for (auto & alt : symbols.alternatives(rest.size() == 1 ? rest[0] : "(" + string_join(rest, " | ") + ")")) {
alts.push_back(std::move(alt));
}
}
if (terminal) {
terminating.chars.insert(terminating.chars.end(), symbols.chars.begin(), symbols.chars.end());
terminating.tokens.insert(terminating.tokens.end(), symbols.tokens.begin(), symbols.tokens.end());
}
}
for (auto & alt : terminating.alternatives()) {
alts.push_back(std::move(alt));
}
for (const auto & [d, symbols] : buckets) {
for (auto & alt : symbols.alternatives(state_name(d))) {
alts.push_back(std::move(alt));
}
}
// every other symbol keeps scanning from the start state
alts.push_back(fallback);
return (optional ? "| " : "") + string_join(alts, " | ");
});
}
// The rules and the tokens named by token rules that the grammar of a parser reaches
struct common_peg_reachable {
std::set<std::string> rules;
std::set<uint32_t> tokens;
};
static void collect_reachable(
const common_peg_arena & arena,
const common_peg_parser_id & root,
common_peg_reachable & reachable
) {
std::set<std::string> visited;
std::function<void(common_peg_parser_id)> visit = [&](common_peg_parser_id id) {
const auto & parser = arena.get(id);
std::visit([&](const auto & p) {
using T = std::decay_t<decltype(p)>;
if constexpr (std::is_same_v<T, common_peg_epsilon_parser> ||
std::is_same_v<T, common_peg_start_parser> ||
std::is_same_v<T, common_peg_end_parser> ||
std::is_same_v<T, common_peg_literal_parser> ||
std::is_same_v<T, common_peg_chars_parser> ||
std::is_same_v<T, common_peg_space_parser> ||
std::is_same_v<T, common_peg_any_parser> ||
std::is_same_v<T, common_peg_string_parser>) {
// These parsers do not have any children
} else if constexpr (std::is_same_v<T, common_peg_token_parser>) {
reachable.tokens.insert(p.token);
} else if constexpr (std::is_same_v<T, common_peg_token_not_parser>) {
reachable.tokens.insert(p.tokens.begin(), p.tokens.end());
} else if constexpr (std::is_same_v<T, common_peg_until_parser>) {
// The delimiters name their tokens in the grammar
if (p.content != COMMON_PEG_INVALID_PARSER_ID) {
visit(p.content);
}
for (const auto & branch : p.branches) {
visit(branch.delimiter);
if (branch.rest != COMMON_PEG_INVALID_PARSER_ID) {
visit(branch.rest);
}
}
} else if constexpr (std::is_same_v<T, common_peg_sequence_parser>) {
for (auto child : p.children) {
visit(child);
}
} else if constexpr (std::is_same_v<T, common_peg_choice_parser>) {
for (auto child : p.children) {
visit(child);
}
} else if constexpr (std::is_same_v<T, common_peg_repetition_parser> ||
std::is_same_v<T, common_peg_and_parser> ||
std::is_same_v<T, common_peg_not_parser> ||
std::is_same_v<T, common_peg_tag_parser> ||
std::is_same_v<T, common_peg_atomic_parser> ||
std::is_same_v<T, common_peg_gbnf_parser> ||
std::is_same_v<T, common_peg_schema_parser>) {
visit(p.child);
} else if constexpr (std::is_same_v<T, common_peg_rule_parser>) {
if (visited.find(p.name) == visited.end()) {
visited.insert(p.name);
reachable.rules.insert(p.name);
visit(p.child);
}
} else if constexpr (std::is_same_v<T, common_peg_ref_parser>) {
// Traverse rules so we pick up everything
auto referenced_rule = arena.get_rule(p.name);
visit(referenced_rule);
} else {
static_assert(is_always_false_v<T>);
}
}, parser);
};
visit(root);
}
// GBNF generation implementation
void common_peg_arena::build_grammar(const common_grammar_builder & builder, bool lazy) const {
common_peg_reachable reachable;
// A raw string value is parsed by the child rather than constrained by the schema
auto schema_delegates = [](const common_peg_schema_parser & s) -> bool {
return !s.node || (s.raw && s.node->may_be_string());
};
// Unwrap the parser so we can properly check if it's a sequence or choice
auto effective_parser = [&](common_peg_parser_id id) -> const common_peg_parser_variant & {
while (true) {
const auto & p = parsers_.at(id);
if (const auto * tag = std::get_if<common_peg_tag_parser>(&p)) {
id = tag->child;
} else if (const auto * atomic = std::get_if<common_peg_atomic_parser>(&p)) {
id = atomic->child;
} else if (const auto * schema = std::get_if<common_peg_schema_parser>(&p)) {
if (schema_delegates(*schema)) {
id = schema->child;
} else {
return p;
}
} else {
return p;
}
}
};
// Generate GBNF for a parser
std::function<std::string(common_peg_parser_id)> to_gbnf = [&](common_peg_parser_id id) -> std::string {
const auto & parser = parsers_.at(id);
return std::visit([&](const auto & p) -> std::string {
using T = std::decay_t<decltype(p)>;
if constexpr (std::is_same_v<T, common_peg_epsilon_parser> ||
std::is_same_v<T, common_peg_start_parser> ||
std::is_same_v<T, common_peg_end_parser>) {
return "";
} else if constexpr (std::is_same_v<T, common_peg_literal_parser>) {
return gbnf_format_literal(p.literal);
} else if constexpr (std::is_same_v<T, common_peg_token_parser>) {
return "<[" + std::to_string(p.token) + "]>";
} else if constexpr (std::is_same_v<T, common_peg_token_not_parser>) {
return gbnf_token_set(std::vector<uint32_t>(p.tokens.begin(), p.tokens.end()), true);
} else if constexpr (std::is_same_v<T, common_peg_sequence_parser>) {
std::string s;
for (const auto & child : p.children) {
auto child_gbnf = to_gbnf(child);
if (child_gbnf.empty()) {
continue;
}
if (!s.empty()) {
s += " ";
}
const auto & child_parser = effective_parser(child);
if (std::holds_alternative<common_peg_choice_parser>(child_parser) ||
std::holds_alternative<common_peg_sequence_parser>(child_parser)) {
s += "(" + child_gbnf + ")";
} else {
s += child_gbnf;
}
}
return s;
} else if constexpr (std::is_same_v<T, common_peg_choice_parser>) {
std::string s;
for (const auto & child : p.children) {
if (!s.empty()) {
s += " | ";
}
auto child_gbnf = to_gbnf(child);
const auto & child_parser = effective_parser(child);
if (std::holds_alternative<common_peg_choice_parser>(child_parser)) {
s += "(" + child_gbnf + ")";
} else {
s += child_gbnf;
}
}
return s;
} else if constexpr (std::is_same_v<T, common_peg_repetition_parser>) {
auto child_gbnf = to_gbnf(p.child);
const auto & child_parser = effective_parser(p.child);
if (std::holds_alternative<common_peg_choice_parser>(child_parser) ||
std::holds_alternative<common_peg_sequence_parser>(child_parser)) {
child_gbnf = "(" + child_gbnf + ")";
}
if (p.min_count == 0 && p.max_count == 1) {
return child_gbnf + "?";
}
if (p.min_count == 0 && p.max_count == -1) {
return child_gbnf + "*";
}
if (p.min_count == 1 && p.max_count == -1) {
return child_gbnf + "+";
}
if (p.max_count == -1) {
return child_gbnf + "{" + std::to_string(p.min_count) + ",}";
}
if (p.min_count == p.max_count) {
if (p.min_count == 1) {
return child_gbnf;
}
return child_gbnf + "{" + std::to_string(p.min_count) + "}";
}
return child_gbnf + "{" + std::to_string(p.min_count) + "," + std::to_string(p.max_count) + "}";
} else if constexpr (std::is_same_v<T, common_peg_and_parser> || std::is_same_v<T, common_peg_not_parser>) {
return ""; // Lookahead not supported in GBNF
} else if constexpr (std::is_same_v<T, common_peg_any_parser>) {
return ".";
} else if constexpr (std::is_same_v<T, common_peg_space_parser>) {
return "space";
} else if constexpr (std::is_same_v<T, common_peg_chars_parser>) {
std::string result = p.pattern;
if (p.min_count == 0 && p.max_count == 1) {
return result + "?";
}
if (p.min_count == 0 && p.max_count == -1) {
return result + "*";
}
if (p.min_count == 1 && p.max_count == -1) {
return result + "+";
}
if (p.max_count == -1) {
return result + "{" + std::to_string(p.min_count) + ",}";
}
if (p.min_count == p.max_count) {
if (p.min_count == 1) {
return result;
}
return result + "{" + std::to_string(p.min_count) + "}";
}
return result + "{" + std::to_string(p.min_count) + "," + std::to_string(p.max_count) + "}";
} else if constexpr (std::is_same_v<T, common_peg_string_parser>) {
const std::string delim(1, p.delimiter);
return R"(( [^)" + delim + R"(\\] | "\\" ( [)" + delim + R"(\\/ bfnrt] | "u" [0-9a-fA-F]{4} ) )*)";
} else if constexpr (std::is_same_v<T, common_peg_until_parser>) {
auto prefix = "until-" + std::to_string(id);
if (!p.include) {
if (p.branches.empty()) {
return ".*";
}
return gbnf_excluding_grammar(builder, prefix, p.matcher, reachable.tokens);
}
std::vector<std::string> branch_rests;
for (size_t b = 0; b < p.branches.size(); b++) {
auto rest = p.branches[b].rest;
branch_rests.push_back(rest == COMMON_PEG_INVALID_PARSER_ID ? "" : builder.add_rule(prefix + "-rest-" + std::to_string(b), to_gbnf(rest)));
}
return gbnf_including_grammar(builder, prefix, p.matcher, reachable.tokens, branch_rests, p.optional);
} else if constexpr (std::is_same_v<T, common_peg_schema_parser>) {
if (schema_delegates(p)) {
return to_gbnf(p.child);
}
return builder.add_schema(p.name, *p.node);
} else if constexpr (std::is_same_v<T, common_peg_rule_parser>) {
return p.name;
} else if constexpr (std::is_same_v<T, common_peg_ref_parser>) {
// Refs should not exist after flattening, but kept just in case
return p.name;
} else if constexpr (std::is_same_v<T, common_peg_tag_parser>) {
return to_gbnf(p.child);
} else if constexpr (std::is_same_v<T, common_peg_atomic_parser>) {
return to_gbnf(p.child);
} else if constexpr (std::is_same_v<T, common_peg_gbnf_parser>) {
return p.grammar;
} else {
static_assert(is_always_false_v<T>);
}
}, parser);
};
// Trigger rules, sorted for a predictable order
std::vector<const common_peg_rule_parser *> trigger_rules;
for (const auto & [name, id] : rules_) {
if (auto rule = std::get_if<common_peg_rule_parser>(&parsers_.at(id))) {
if (rule->trigger) {
trigger_rules.push_back(rule);
}
}
}
std::sort(trigger_rules.begin(), trigger_rules.end(), [](const auto * a, const auto * b) { return a->name < b->name; });
// A lazy grammar scans for the starts when the trigger rules were built from a start and rest
bool scans = false;
for (const auto * rule : trigger_rules) {
scans = scans || rule->start != COMMON_PEG_INVALID_PARSER_ID;
}
if (scans) {
for (const auto * rule : trigger_rules) {
if (rule->start == COMMON_PEG_INVALID_PARSER_ID) {
throw std::runtime_error("trigger rule without a start in a scanning grammar: " + rule->name);
}
}
}
if (lazy && scans) {
for (const auto * rule : trigger_rules) {
collect_reachable(*this, rule->start, reachable);
collect_reachable(*this, rule->rest, reachable);
}
} else if (lazy) {
for (const auto * rule : trigger_rules) {
collect_reachable(*this, rules_.at(rule->name), reachable);
}
} else {
collect_reachable(*this, root_, reachable);
}
// Create GBNF rules for all reachable rules
for (const auto & [name, rule_id] : rules_) {
if (reachable.rules.find(name) == reachable.rules.end()) {
continue;
}
const auto & parser = parsers_.at(rule_id);
if (auto rule = std::get_if<common_peg_rule_parser>(&parser)) {
builder.add_rule(rule->name, to_gbnf(rule->child));
}
}
if (lazy && scans) {
// The root scans for a trigger start and then constrains its rest
common_trie starts;
std::vector<std::string> rests;
for (const auto * rule : trigger_rules) {
auto value = (int32_t) rests.size();
rests.push_back(builder.add_rule(rule->name + "-rest", to_gbnf(rule->rest)));
for (size_t end : insert_delimiter(starts, *this, rule->start)) {
auto owner = starts.mark(end, value);
if (owner != value) {
throw std::runtime_error("trigger rules " + trigger_rules[owner]->name + " and " + rule->name + " must not share a start");
}
}
}
builder.add_rule("root", gbnf_including_grammar(builder, "trigger", starts, reachable.tokens, rests, /* optional = */ true));
} else if (lazy) {
// Generate root rule from trigger rules only
std::vector<std::string> trigger_names;
for (const auto * rule : trigger_rules) {
trigger_names.push_back(rule->name);
}
builder.add_rule("root", string_join(trigger_names, " | "));
} else if (root_ != COMMON_PEG_INVALID_PARSER_ID) {
builder.add_rule("root", to_gbnf(root_));
}
}
common_peg_arena build_peg_parser(const std::function<common_peg_parser(common_peg_parser_builder & builder)> & fn) {
common_peg_parser_builder builder;
builder.set_root(fn(builder));
return builder.build();
}