#include "peg-parser.h" #include "common.h" #include "json-schema-to-grammar.h" #include "log.h" #include "trie.h" #include "unicode.h" #include #include #include #include #include #include // The parser can run with a lower limit than COMMON_RECURSIVE_LIMIT static constexpr int RULE_RECURSIVE_LIMIT = 64; // Trick to catch missing branches template inline constexpr bool is_always_false_v = false; const char * common_peg_parse_result_type_name(common_peg_parse_result_type type) { switch (type) { case COMMON_PEG_PARSE_RESULT_FAIL: return "fail"; case COMMON_PEG_PARSE_RESULT_SUCCESS: return "success"; case COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT: return "need_more_input"; default: return "unknown"; } } static bool is_hex_digit(const char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } static std::pair parse_hex_escape(const std::string & str, size_t pos, int hex_count) { if (pos + hex_count > str.length()) { return {0, 0}; } uint32_t value = 0; for (int i = 0; i < hex_count; i++) { char c = str[pos + i]; if (!is_hex_digit(c)) { return {0, 0}; } value <<= 4; if ('a' <= c && c <= 'f') { value += c - 'a' + 10; } else if ('A' <= c && c <= 'F') { value += c - 'A' + 10; } else if ('0' <= c && c <= '9') { value += c - '0'; } else { break; } } return {value, static_cast(hex_count)}; } static std::pair parse_char_class_char(const std::string & content, size_t pos) { if (content[pos] == '\\' && pos + 1 < content.length()) { switch (content[pos + 1]) { case 'x': { auto result = parse_hex_escape(content, pos + 2, 2); if (result.second > 0) { return {result.first, 2 + result.second}; } // Invalid escape, treat as literal 'x' return {static_cast('x'), 2}; } case 'u': { auto result = parse_hex_escape(content, pos + 2, 4); if (result.second > 0) { return {result.first, 2 + result.second}; } // Invalid escape, treat as literal 'u' return {static_cast('u'), 2}; } case 'U': { auto result = parse_hex_escape(content, pos + 2, 8); if (result.second > 0) { return {result.first, 2 + result.second}; } // Invalid escape, treat as literal 'U' return {static_cast('U'), 2}; } case 'n': return {'\n', 2}; case 't': return {'\t', 2}; case 'r': return {'\r', 2}; case '\\': return {'\\', 2}; case ']': return {']', 2}; case '[': return {'[', 2}; default: return {static_cast(content[pos + 1]), 2}; } } // Regular character - return as codepoint return {static_cast(static_cast(content[pos])), 1}; } static std::pair, bool> parse_char_classes(const std::string & classes) { std::vector ranges; bool negated = false; std::string content = classes; if (content.front() == '[') { content = content.substr(1); } if (content.back() == ']') { content.pop_back(); } // Check for negation if (!content.empty() && content.front() == '^') { negated = true; content = content.substr(1); } size_t i = 0; while (i < content.length()) { auto [start, start_len] = parse_char_class_char(content, i); i += start_len; if (i + 1 < content.length() && content[i] == '-') { // Range detected auto [end, end_len] = parse_char_class_char(content, i + 1); ranges.push_back(common_peg_chars_parser::char_range{start, end}); i += 1 + end_len; } else { ranges.push_back(common_peg_chars_parser::char_range{start, start}); } } return {ranges, negated}; } common_peg_ast_id common_peg_ast_arena::find_by_tag(const common_peg_ast_node & parent, const std::string & tag, int max_depth) const { for (auto child_id : parent.children) { const auto & child = get(child_id); if (child.tag == tag) { return child_id; } if (max_depth > 1) { auto result = find_by_tag(child, tag, max_depth - 1); if (result != COMMON_PEG_INVALID_AST_ID) { return result; } } } return COMMON_PEG_INVALID_AST_ID; } common_peg_ast_id common_peg_ast_arena::find_by_rule(const common_peg_ast_node & parent, const std::string & rule, int max_depth) const { for (auto child_id : parent.children) { const auto & child = get(child_id); if (child.rule == rule) { return child_id; } if (max_depth > 1) { auto result = find_by_rule(child, rule, max_depth - 1); if (result != COMMON_PEG_INVALID_AST_ID) { return result; } } } return COMMON_PEG_INVALID_AST_ID; } std::string common_peg_ast_node::sanitized_text() const { if (invalid_utf8.empty()) { return std::string(text); } std::string out; out.reserve(text.size() + 2 * invalid_utf8.size()); size_t seg_start = start; for (const auto & invalid : invalid_utf8) { out.append(text.data() + (seg_start - start), invalid.pos - seg_start); out.append("\xEF\xBF\xBD"); seg_start = invalid.pos + invalid.len; } out.append(text.data() + (seg_start - start), end - seg_start); return out; } void common_peg_ast_arena::visit(common_peg_ast_id id, const common_peg_ast_visitor & visitor) const { // no recursion: the AST is several nodes deep per level of the input std::vector stack = { id }; while (!stack.empty()) { const auto cur = stack.back(); stack.pop_back(); if (cur == COMMON_PEG_INVALID_AST_ID) { continue; } const auto & node = get(cur); visitor(node); stack.insert(stack.end(), node.children.rbegin(), node.children.rend()); } } void common_peg_ast_arena::visit(const common_peg_parse_result & result, const common_peg_ast_visitor & visitor) const { for (const auto & node : result.nodes) { visit(node, visitor); } } common_peg_tokens::common_peg_tokens(const llama_vocab * vocab) { const llama_token n_tokens = llama_vocab_n_tokens(vocab); for (llama_token id = 0; id < n_tokens; ++id) { const auto attr = llama_vocab_get_attr(vocab, id); if (attr & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED)) { std::string text = llama_vocab_get_text(vocab, id); ids.emplace(text, id); tokens.emplace(id, common_peg_token{ std::move(text), attr }); } } } llama_token common_peg_tokens::token_id(const std::string & text) const { auto it = ids.find(text); return it != ids.end() ? it->second : LLAMA_TOKEN_NULL; } // Insert every delimiter the parser matches into the trie, starting from each node in from. Returns the end nodes. // A choice adds the delimiters of each branch. A sequence joins every combination of its children. static std::set insert_delimiter(common_trie & trie, const common_peg_arena & arena, common_peg_parser_id id, std::set from = { 0 }) { return std::visit([&](const auto & p) -> std::set { using T = std::decay_t; if constexpr (std::is_same_v || std::is_same_v) { return insert_delimiter(trie, arena, p.child, std::move(from)); } else if constexpr (std::is_same_v) { std::set ends; for (size_t node : from) { size_t pos = 0; while (pos < p.literal.size()) { auto result = common_parse_utf8_codepoint(p.literal, pos); if (result.status != utf8_parse_result::SUCCESS) { break; } node = trie.extend(node, common_trie::symbol::codepoint(result.codepoint)); pos += result.bytes_consumed; } ends.insert(node); } return ends; } else if constexpr (std::is_same_v) { std::set ends; for (size_t node : from) { ends.insert(trie.extend(node, common_trie::symbol::token(p.token))); } return ends; } else if constexpr (std::is_same_v) { std::set ends; for (auto child : p.children) { ends.merge(insert_delimiter(trie, arena, child, from)); } return ends; } else if constexpr (std::is_same_v) { for (auto child : p.children) { from = insert_delimiter(trie, arena, child, std::move(from)); } return from; } else { throw std::invalid_argument("a delimiter may only be a sequence of literals, tokens, and choices of them, possibly wrapped in tags and atomics"); } }, arena.get(id)); } // Build the matcher of an until parser, each pattern valued by the branch it belongs to static common_trie build_until_matcher(const common_peg_arena & arena, const std::vector & branches, bool include) { common_trie matcher; for (size_t b = 0; b < branches.size(); b++) { for (size_t end : insert_delimiter(matcher, arena, branches[b].delimiter)) { if (end == 0) { throw std::invalid_argument("delimiter must not match the empty string"); } // nothing follows a delimiter without include, so the first branch to claim it wins auto owner = matcher.mark(end, (int32_t) b); if (owner != (int32_t) b && include) { throw std::invalid_argument("until branches " + std::to_string(owner) + " and " + std::to_string(b) + " must not share a delimiter"); } } } return matcher; } struct parser_executor; common_peg_parser_id common_peg_arena::add_parser(common_peg_parser_variant parser) { common_peg_parser_id id = parsers_.size(); parsers_.push_back(std::move(parser)); return id; } void common_peg_arena::add_rule(const std::string & name, common_peg_parser_id id) { rules_[name] = id; } common_peg_parser_id common_peg_arena::get_rule(const std::string & name) const { auto it = rules_.find(name); if (it == rules_.end()) { throw std::runtime_error("Rule not found: " + name); } return it->second; } struct parser_executor { const common_peg_arena & arena; common_peg_parse_context & ctx; size_t start_pos; parser_executor(const common_peg_arena & arena, common_peg_parse_context & ctx, size_t start) : arena(arena), ctx(ctx), start_pos(start) {} std::string debug_indent() const { return std::string(ctx.parse_depth * 2, ' '); } std::string debug_input_snippet(size_t pos, size_t len = 60) const { if (pos >= ctx.input.size()) { return ""; } auto snippet = ctx.input.substr(pos, len); // Escape newlines for display std::string result; for (char c : snippet) { if (c == '\n') { result += "\\n"; } else if (c == '\r') { result += "\\r"; } else if (c == '\t') { result += "\\t"; } else { result += c; } } if (pos + len < ctx.input.size()) { result += "..."; } return result; } common_peg_parse_result operator()(const common_peg_epsilon_parser & /* p */) const { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos); } common_peg_parse_result operator()(const common_peg_start_parser & /* p */) const { return common_peg_parse_result( start_pos == 0 ? COMMON_PEG_PARSE_RESULT_SUCCESS : COMMON_PEG_PARSE_RESULT_FAIL, start_pos ); } common_peg_parse_result operator()(const common_peg_end_parser & /* p */) const { return common_peg_parse_result( start_pos >= ctx.input.size() ? COMMON_PEG_PARSE_RESULT_SUCCESS : COMMON_PEG_PARSE_RESULT_FAIL, start_pos ); } common_peg_parse_result operator()(const common_peg_literal_parser & p) { auto pos = start_pos; for (auto i = 0u; i < p.literal.size(); ++i) { if (pos >= ctx.input.size()) { if (!ctx.is_lenient()) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); } if (ctx.input[pos] != p.literal[i]) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } ++pos; } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); } common_peg_parse_result operator()(const common_peg_token_parser & p) { if (start_pos >= ctx.input.size()) { if (!ctx.is_lenient()) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos); } if (start_pos >= ctx.tokens.size() || ctx.tokens[start_pos] != p.token) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } // Skip the rest of the piece, up to the next token or the end of the input auto pos = start_pos + 1; while (pos < ctx.input.size() && ctx.tokens[pos] == LLAMA_TOKEN_NULL) { ++pos; } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); } common_peg_parse_result operator()(const common_peg_token_not_parser & p) { if (start_pos >= ctx.input.size()) { if (!ctx.is_lenient()) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos); } if (start_pos >= ctx.tokens.size() || ctx.tokens[start_pos] == LLAMA_TOKEN_NULL || std::find(p.tokens.begin(), p.tokens.end(), ctx.tokens[start_pos]) != p.tokens.end()) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } // Skip the rest of the piece, up to the next token or the end of the input auto pos = start_pos + 1; while (pos < ctx.input.size() && ctx.tokens[pos] == LLAMA_TOKEN_NULL) { ++pos; } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); } common_peg_parse_result operator()(const common_peg_sequence_parser & p) { if (ctx.is_debug()) { LOG_DBG("%sSEQ start at %zu '%s' (%zu children)\n", debug_indent().c_str(), start_pos, debug_input_snippet(start_pos).c_str(), p.children.size()); } ctx.parse_depth++; auto pos = start_pos; std::vector nodes; std::vector invalid_utf8; for (size_t i = 0; i < p.children.size(); i++) { const auto & child_id = p.children[i]; if (ctx.is_debug()) { fprintf(stderr, "%sSEQ child %zu: %s\n", debug_indent().c_str(), i, arena.dump(child_id).c_str()); } auto result = arena.parse(child_id, ctx, pos); if (ctx.is_debug()) { fprintf(stderr, "%sSEQ child %zu: %s at %zu->%zu\n", debug_indent().c_str(), i, common_peg_parse_result_type_name(result.type), result.start, result.end); } if (result.fail()) { ctx.parse_depth--; if (ctx.is_debug()) { fprintf(stderr, "%sSEQ -> FAIL\n", debug_indent().c_str()); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos, result.end); } if (!result.nodes.empty()) { nodes.insert(nodes.end(), result.nodes.begin(), result.nodes.end()); } invalid_utf8.insert(invalid_utf8.end(), result.invalid_utf8.begin(), result.invalid_utf8.end()); if (result.need_more_input()) { ctx.parse_depth--; if (ctx.is_debug()) { fprintf(stderr, "%sSEQ -> NEED_MORE\n", debug_indent().c_str()); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, result.end, std::move(nodes), std::move(invalid_utf8)); } pos = result.end; } ctx.parse_depth--; if (ctx.is_debug()) { fprintf(stderr, "%sSEQ -> SUCCESS at %zu->%zu\n", debug_indent().c_str(), start_pos, pos); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, std::move(nodes), std::move(invalid_utf8)); } common_peg_parse_result operator()(const common_peg_choice_parser & p) { if (ctx.is_debug()) { fprintf(stderr, "%sCHOICE start at %zu '%s' (%zu options)\n", debug_indent().c_str(), start_pos, debug_input_snippet(start_pos).c_str(), p.children.size()); } ctx.parse_depth++; auto pos = start_pos; for (size_t i = 0; i < p.children.size(); i++) { const auto & child_id = p.children[i]; if (ctx.is_debug()) { fprintf(stderr, "%sCHOICE option %zu: %s\n", debug_indent().c_str(), i, arena.dump(child_id).c_str()); } auto result = arena.parse(child_id, ctx, pos); if (ctx.is_debug()) { fprintf(stderr, "%sCHOICE option %zu: %s\n", debug_indent().c_str(), i, common_peg_parse_result_type_name(result.type)); } if (!result.fail()) { ctx.parse_depth--; if (ctx.is_debug()) { fprintf(stderr, "%sCHOICE -> %s (option %zu)\n", debug_indent().c_str(), common_peg_parse_result_type_name(result.type), i); } return result; } } ctx.parse_depth--; if (ctx.is_debug()) { fprintf(stderr, "%sCHOICE -> FAIL (no options matched)\n", debug_indent().c_str()); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } common_peg_parse_result operator()(const common_peg_repetition_parser & p) { if (ctx.is_debug()) { fprintf(stderr, "%sREPEAT start at %zu '%s' (min=%d, max=%d)\n", debug_indent().c_str(), start_pos, debug_input_snippet(start_pos).c_str(), p.min_count, p.max_count); } ctx.parse_depth++; auto pos = start_pos; int match_count = 0; std::vector nodes; std::vector invalid_utf8; // Try to match up to max_count times (or unlimited if max_count is -1) while (p.max_count == -1 || match_count < p.max_count) { if (pos >= ctx.input.size()) { if (ctx.is_debug()) { fprintf(stderr, "%sREPEAT: at end of input, count=%d\n", debug_indent().c_str(), match_count); } break; } auto result = arena.parse(p.child, ctx, pos); if (ctx.is_debug()) { fprintf(stderr, "%sREPEAT iter %d: %s at %zu->%zu, nodes=%zu\n", debug_indent().c_str(), match_count, common_peg_parse_result_type_name(result.type), result.start, result.end, result.nodes.size()); fprintf(stderr, "%sREPEAT CHILD: %s\n", debug_indent().c_str(), arena.dump(p.child).c_str()); } if (result.success()) { // Prevent infinite loop on empty matches if (result.end == pos) { if (ctx.is_debug()) { fprintf(stderr, "%s REPEAT: empty match, stopping\n", debug_indent().c_str()); } break; } if (!result.nodes.empty()) { nodes.insert(nodes.end(), result.nodes.begin(), result.nodes.end()); } invalid_utf8.insert(invalid_utf8.end(), result.invalid_utf8.begin(), result.invalid_utf8.end()); pos = result.end; match_count++; continue; } if (result.need_more_input()) { if (!result.nodes.empty()) { nodes.insert(nodes.end(), result.nodes.begin(), result.nodes.end()); } invalid_utf8.insert(invalid_utf8.end(), result.invalid_utf8.begin(), result.invalid_utf8.end()); ctx.parse_depth--; if (ctx.is_debug()) { fprintf(stderr, "%sREPEAT -> NEED_MORE (count=%d, nodes=%zu)\n", debug_indent().c_str(), match_count, nodes.size()); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, result.end, std::move(nodes), std::move(invalid_utf8)); } // Child failed - stop trying if (ctx.is_debug()) { fprintf(stderr, "%sREPEAT: child failed, stopping\n", debug_indent().c_str()); } break; } // Check if we got enough matches if (p.min_count > 0 && match_count < p.min_count) { ctx.parse_depth--; if (pos >= ctx.input.size() && ctx.is_lenient()) { if (ctx.is_debug()) { fprintf(stderr, "%sREPEAT -> NEED_MORE (not enough matches: %d < %d)\n", debug_indent().c_str(), match_count, p.min_count); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos, std::move(nodes), std::move(invalid_utf8)); } if (ctx.is_debug()) { fprintf(stderr, "%sREPEAT -> FAIL (not enough matches: %d < %d)\n", debug_indent().c_str(), match_count, p.min_count); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos, pos); } ctx.parse_depth--; if (ctx.is_debug()) { fprintf(stderr, "%sREPEAT -> SUCCESS (count=%d, nodes=%zu)\n", debug_indent().c_str(), match_count, nodes.size()); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, std::move(nodes), std::move(invalid_utf8)); } common_peg_parse_result operator()(const common_peg_and_parser & p) { auto result = arena.parse(p.child, ctx, start_pos); // Pass result but don't consume input return common_peg_parse_result(result.type, start_pos); } common_peg_parse_result operator()(const common_peg_not_parser & p) { auto result = arena.parse(p.child, ctx, start_pos); if (result.success()) { // Fail if the underlying parser matches return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } if (result.need_more_input()) { // Propagate - need to know what child would match before negating return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos); } // Child failed, so negation succeeds return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos); } common_peg_parse_result operator()(const common_peg_any_parser & /* p */) const { // Parse a single UTF-8 codepoint (not just a single byte) auto result = common_parse_utf8_codepoint(ctx.input, start_pos); if (result.status == utf8_parse_result::INCOMPLETE) { if (!ctx.is_lenient()) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos); } if (result.status == utf8_parse_result::INVALID) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, start_pos + result.bytes_consumed); } common_peg_parse_result operator()(const common_peg_space_parser & /* p */) { auto pos = start_pos; while (pos < ctx.input.size()) { auto c = static_cast(ctx.input[pos]); if (std::isspace(c)) { ++pos; } else { break; } } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); } common_peg_parse_result operator()(const common_peg_chars_parser & p) const { auto pos = start_pos; int match_count = 0; // Try to match up to max_count times (or unlimited if max_count is -1) while (p.max_count == -1 || match_count < p.max_count) { auto result = common_parse_utf8_codepoint(ctx.input, pos); if (result.status == utf8_parse_result::INCOMPLETE) { if (match_count >= p.min_count) { // We have enough matches, succeed with what we have return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); } // Not enough matches yet if (!ctx.is_lenient()) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); } if (result.status == utf8_parse_result::INVALID) { // Malformed UTF-8 in input if (match_count >= p.min_count) { // We have enough matches, succeed up to here return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); } // Not enough matches, fail return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } // Check if this codepoint matches our character class bool matches = false; for (const auto & range : p.ranges) { if (range.contains(result.codepoint)) { matches = true; break; } } // If negated, invert the match result if (p.negated) { matches = !matches; } if (matches) { pos += result.bytes_consumed; ++match_count; } else { // Character doesn't match, stop matching break; } } // Check if we got enough matches if (match_count < p.min_count) { if (pos >= ctx.input.size() && ctx.is_lenient()) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos, pos); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); } static common_peg_parse_result handle_escape_sequence(common_peg_parse_context & ctx, size_t start, size_t & pos, const char delimiter) { auto save = pos; ++pos; // consume '\' if (pos >= ctx.input.size()) { if (!ctx.is_lenient()) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start); } pos = save; // suppress unmatched '\' return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start, pos); } char c = ctx.input[pos]; if (c == delimiter || c == '\\' || c == '/' || c == 'b' || c == 'f' || c == 'n' || c == 'r' || c == 't') { ++pos; return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start, pos); } if (c == 'u') { auto result = handle_unicode_escape(ctx, start, pos); if (result.need_more_input()) { pos = save; // suppress incomplete sequence return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start, pos); } return result; } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start); } static common_peg_parse_result handle_unicode_escape(common_peg_parse_context & ctx, size_t start, size_t & pos) { ++pos; // consume 'u' for (int i = 0; i < 4; ++i) { if (pos >= ctx.input.size()) { if (!ctx.is_lenient()) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start, pos); } if (!is_hex_digit(ctx.input[pos])) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start); } ++pos; } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start, pos); } common_peg_parse_result operator()(const common_peg_string_parser & p) { auto pos = start_pos; // Parse string content (without quotes) while (pos < ctx.input.size()) { char c = ctx.input[pos]; if (c == p.delimiter) { // Found closing delimiter - success (don't consume it) return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); } if (c == '\\') { auto result = handle_escape_sequence(ctx, start_pos, pos, p.delimiter); if (!result.success()) { return result; } } else { auto utf8_result = common_parse_utf8_codepoint(ctx.input, pos); if (utf8_result.status == utf8_parse_result::INCOMPLETE) { if (!ctx.is_lenient()) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); } if (utf8_result.status == utf8_parse_result::INVALID) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } pos += utf8_result.bytes_consumed; } } // Reached end without finding closing quote if (!ctx.is_lenient()) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos, pos); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); } common_peg_parse_result operator()(const common_peg_until_parser & p) const { if (p.include) { return parse_through(p); } const auto & matcher = p.matcher; // Scan input and check for delimiters size_t pos = start_pos; size_t last_valid_pos = start_pos; std::vector invalid_utf8; while (pos < ctx.input.size()) { auto step = matcher.next(ctx.input, ctx.tokens, pos); if (step.status == utf8_parse_result::INCOMPLETE && ctx.is_lenient()) { // The rest of the sequence may still arrive, return what we have so far return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, last_valid_pos, {}, std::move(invalid_utf8)); } if (step.status != utf8_parse_result::SUCCESS) { // Malformed UTF-8, or a sequence truncated by the end of a complete input. // A delimiter cannot start inside bytes that fail to decode, so consume them and move on invalid_utf8.push_back({pos, step.bytes_consumed}); pos += step.bytes_consumed; last_valid_pos = pos; continue; } // Check if a delimiter starts at this position auto match = matcher.check_at(ctx.input, ctx.tokens, pos); if (match == common_trie::COMPLETE_MATCH) { // Found a complete delimiter, return everything before it return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, {}, std::move(invalid_utf8)); } if (match == common_trie::PARTIAL_MATCH && !ctx.is_complete()) { // Found a partial match extending to end of input, return everything before it. // On a complete input the delimiter can no longer form, so the bytes belong to the text return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, {}, std::move(invalid_utf8)); } pos += step.bytes_consumed; last_valid_pos = pos; } if (last_valid_pos == ctx.input.size() && ctx.is_lenient()) { // Reached the end of a partial stream, there might still be more input that we need to consume. return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, last_valid_pos, {}, std::move(invalid_utf8)); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, last_valid_pos, {}, std::move(invalid_utf8)); } // Parse content, then the first branch whose delimiter and rest follow it, like C (D1 R1 | D2 R2 | ...) common_peg_parse_result parse_through(const common_peg_until_parser & p) const { auto content = arena.parse(p.content, ctx, start_pos); if (content.fail() || content.need_more_input()) { return content; } auto pos = content.end; for (const auto & branch : p.branches) { auto nodes = content.nodes; auto invalid_utf8 = content.invalid_utf8; auto end = pos; bool matched = true; for (auto id : { branch.delimiter, branch.rest }) { if (id == COMMON_PEG_INVALID_PARSER_ID) { continue; } auto result = arena.parse(id, ctx, end); if (result.fail()) { matched = false; break; } nodes.insert(nodes.end(), result.nodes.begin(), result.nodes.end()); invalid_utf8.insert(invalid_utf8.end(), result.invalid_utf8.begin(), result.invalid_utf8.end()); end = result.end; if (result.need_more_input()) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, end, std::move(nodes), std::move(invalid_utf8)); } } if (matched) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, end, std::move(nodes), std::move(invalid_utf8)); } } if (p.optional && pos == ctx.input.size()) { auto type = ctx.is_lenient() ? COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT : COMMON_PEG_PARSE_RESULT_SUCCESS; return common_peg_parse_result(type, start_pos, pos, std::move(content.nodes), std::move(content.invalid_utf8)); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos, pos); } common_peg_parse_result operator()(const common_peg_schema_parser & p) { return arena.parse(p.child, ctx, start_pos); } common_peg_parse_result operator()(const common_peg_rule_parser & p) { // only rules recurse, a recursive rule is entered once per input level if (ctx.rule_depth[p.child] >= RULE_RECURSIVE_LIMIT) { throw common_peg_recursion_error("input is nested more than " + std::to_string(RULE_RECURSIVE_LIMIT) + " levels deep"); } // Parse the child ctx.rule_depth[p.child]++; auto result = arena.parse(p.child, ctx, start_pos); ctx.rule_depth[p.child]--; if (!result.fail()) { std::string_view text; if (result.start < ctx.input.size()) { text = std::string_view(ctx.input).substr(result.start, result.end - result.start); } auto node_id = ctx.ast.add_node( p.name, "", result.start, result.end, text, std::move(result.nodes), result.need_more_input(), result.invalid_utf8 ); return common_peg_parse_result(result.type, result.start, result.end, { node_id }, std::move(result.invalid_utf8)); } return result; } common_peg_parse_result operator()(const common_peg_tag_parser & p) { // Parse the child if (ctx.is_debug()) { fprintf(stderr, "%sTAG: %s\n", debug_indent().c_str(), p.tag.c_str()); } auto result = arena.parse(p.child, ctx, start_pos); if (!result.fail()) { std::string_view text; if (result.start < ctx.input.size()) { text = std::string_view(ctx.input).substr(result.start, result.end - result.start); } auto node_id = ctx.ast.add_node( "", p.tag, result.start, result.end, text, std::move(result.nodes), result.need_more_input(), result.invalid_utf8 ); return common_peg_parse_result(result.type, result.start, result.end, { node_id }, std::move(result.invalid_utf8)); } return result; } common_peg_parse_result operator()(const common_peg_ref_parser & p) { auto rule_id = arena.get_rule(p.name); return arena.parse(rule_id, ctx, start_pos); } common_peg_parse_result operator()(const common_peg_atomic_parser & p) { auto result = arena.parse(p.child, ctx, start_pos); if (result.need_more_input()) { // Clear nodes so they don't propagate up. result.nodes.clear(); } return result; } common_peg_parse_result operator()(const common_peg_gbnf_parser & p) { return arena.parse(p.child, ctx, start_pos); } }; common_peg_parse_result common_peg_arena::parse(common_peg_parse_context & ctx, size_t start) const { if (root_ == COMMON_PEG_INVALID_PARSER_ID) { throw std::runtime_error("No root parser set"); } // Initialize rule depth for all parsers ctx.rule_depth.assign(parsers_.size(), 0); return parse(root_, ctx, start); } common_peg_parse_result common_peg_arena::parse(common_peg_parser_id id, common_peg_parse_context & ctx, size_t start) const { // Execute parser const auto & parser = parsers_.at(id); parser_executor exec(*this, ctx, start); return std::visit(exec, parser); } common_peg_parser_id common_peg_arena::resolve_ref(common_peg_parser_id id) { const auto & parser = parsers_.at(id); if (auto ref = std::get_if(&parser)) { return get_rule(ref->name); } return id; } static void bfs_node(common_peg_ast_arena &arena, std::ostringstream & oss, const common_peg_ast_node & node, int indent) { for (int i = 0; i < indent; i++) { oss << " "; } oss << "NODE " << node.id; if (!node.rule.empty()) { oss << " (rule " << node.rule << ")"; } if (!node.tag.empty()) { oss << " (tag " << node.tag << ")"; } oss << " ['" << node.text << "']\n"; for (const auto child : node.children) { bfs_node(arena, oss, arena.get(child), indent + 1); } } std::string common_peg_ast_arena::dump() { std::ostringstream oss; for (auto & node : nodes_) { bfs_node(*this, oss, node, 0); } return oss.str(); } void common_peg_arena::resolve_refs() { // Walk through all parsers and replace refs with their corresponding rule IDs for (auto & parser : parsers_) { std::visit([this](auto & p) { using T = std::decay_t; if constexpr (std::is_same_v) { for (auto & child : p.children) { child = resolve_ref(child); } } else if constexpr (std::is_same_v) { for (auto & child : p.children) { child = resolve_ref(child); } } else if constexpr (std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v) { p.child = resolve_ref(p.child); } else if constexpr (std::is_same_v) { if (p.content != COMMON_PEG_INVALID_PARSER_ID) { p.content = resolve_ref(p.content); } for (auto & branch : p.branches) { branch.delimiter = resolve_ref(branch.delimiter); if (branch.rest != COMMON_PEG_INVALID_PARSER_ID) { branch.rest = resolve_ref(branch.rest); } } } else if constexpr (std::is_same_v) { p.child = resolve_ref(p.child); if (p.start != COMMON_PEG_INVALID_PARSER_ID) { p.start = resolve_ref(p.start); p.rest = resolve_ref(p.rest); } } else if constexpr (std::is_same_v) { p.child = resolve_ref(p.child); } else if constexpr (std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v) { // These rules do not have children } else { static_assert(is_always_false_v); } }, parser); } // Also flatten root if it's a ref if (root_ != COMMON_PEG_INVALID_PARSER_ID) { root_ = resolve_ref(root_); } } std::string common_peg_arena::dump(common_peg_parser_id id) const { std::set visited; return dump_impl(id, visited); } std::string common_peg_arena::dump_impl(common_peg_parser_id id, std::set & visited) const { // Check for cycles if (visited.count(id)) { return "[cycle]"; } visited.insert(id); const auto & parser = parsers_.at(id); return std::visit([this, &visited](const auto & p) -> std::string { using T = std::decay_t; if constexpr (std::is_same_v) { return "Epsilon"; } else if constexpr (std::is_same_v) { return "Start"; } else if constexpr (std::is_same_v) { return "End"; } else if constexpr (std::is_same_v) { return "Literal(" + p.literal + ")"; } else if constexpr (std::is_same_v) { return "Token(" + p.piece + ", " + std::to_string(p.token) + ")"; } else if constexpr (std::is_same_v) { std::vector parts; for (auto token : p.tokens) { parts.push_back(std::to_string(token)); } return "TokenNot(" + string_join(parts, ", ") + ")"; } else if constexpr (std::is_same_v) { std::vector parts; for (const auto & child : p.children) { parts.push_back(dump_impl(child, visited)); } return "Sequence(" + string_join(parts, ", ") + ")"; } else if constexpr (std::is_same_v) { std::vector parts; for (const auto & child : p.children) { parts.push_back(dump_impl(child, visited)); } return "Choice(" + string_join(parts, ", ") + ")"; } else if constexpr (std::is_same_v) { if (p.max_count == -1) { return "Repetition(" + dump_impl(p.child, visited) + ", " + std::to_string(p.min_count) + ", unbounded)"; } return "Repetition(" + dump_impl(p.child, visited) + ", " + std::to_string(p.min_count) + ", " + std::to_string(p.max_count) + ")"; } else if constexpr (std::is_same_v) { return "And(" + dump_impl(p.child, visited) + ")"; } else if constexpr (std::is_same_v) { return "Not(" + dump_impl(p.child, visited) + ")"; } else if constexpr (std::is_same_v) { return "Atomic(" + dump_impl(p.child, visited) + ")"; } else if constexpr (std::is_same_v) { return "Gbnf(" + p.grammar + ", " + dump_impl(p.child, visited) + ")"; } else if constexpr (std::is_same_v) { return "Any"; } else if constexpr (std::is_same_v) { return "Space"; } else if constexpr (std::is_same_v) { if (p.max_count == -1) { return "CharRepeat(" + p.pattern + ", " + std::to_string(p.min_count) + ", unbounded)"; } return "CharRepeat(" + p.pattern + ", " + std::to_string(p.min_count) + ", " + std::to_string(p.max_count) + ")"; } else if constexpr (std::is_same_v) { return "String(" + std::string(1, p.delimiter) + ")"; } else if constexpr (std::is_same_v) { std::vector branches; for (const auto & branch : p.branches) { auto text = dump_impl(branch.delimiter, visited); if (branch.rest != COMMON_PEG_INVALID_PARSER_ID) { text += " " + dump_impl(branch.rest, visited); } branches.push_back(std::move(text)); } if (p.include) { return "Until(" + dump_impl(p.content, visited) + ", " + string_join(branches, " | ") + (p.optional ? ", optional" : "") + ")"; } return "Until(" + string_join(branches, " | ") + ")"; } else if constexpr (std::is_same_v) { return "Schema(" + dump_impl(p.child, visited) + ", " + (p.node ? common_chat_schema::kind_name(p.node->kind()) : "null") + ")"; } else if constexpr (std::is_same_v) { return "Rule(" + p.name + ", " + dump_impl(p.child, visited) + ")"; } else if constexpr (std::is_same_v) { return "Ref(" + p.name + ")"; } else if constexpr (std::is_same_v) { return "Tag(" + p.tag + ", " + dump_impl(p.child, visited) + ")"; } else if constexpr (std::is_same_v) { return "Atomic(" + dump_impl(p.child, visited) + ")"; } else { return "Unknown"; } }, parser); } common_peg_parser & common_peg_parser::operator=(const common_peg_parser & other) { id_ = other.id_; return *this; } common_peg_parser & common_peg_parser::operator+=(const common_peg_parser & other) { id_ = builder_.sequence({id_, other.id_}); return *this; } common_peg_parser & common_peg_parser::operator|=(const common_peg_parser & other) { id_ = builder_.choice({id_, other.id_}); return *this; } common_peg_parser common_peg_parser::operator+(const common_peg_parser & other) const { return builder_.sequence({id_, other.id_}); } common_peg_parser common_peg_parser::operator|(const common_peg_parser & other) const { return builder_.choice({id_, other.id_}); } common_peg_parser common_peg_parser::operator<<(const common_peg_parser & other) const { return builder_.sequence({id_, builder_.space(), other.id_}); } common_peg_parser common_peg_parser::operator+(const char * str) const { return *this + builder_.literal(str); } common_peg_parser common_peg_parser::operator+(const std::string & str) const { return *this + builder_.literal(str); } common_peg_parser common_peg_parser::operator<<(const char * str) const { return *this << builder_.literal(str); } common_peg_parser common_peg_parser::operator<<(const std::string & str) const { return *this << builder_.literal(str); } common_peg_parser common_peg_parser::operator|(const char * str) const { return *this | builder_.literal(str); } common_peg_parser common_peg_parser::operator|(const std::string & str) const { return *this | builder_.literal(str); } common_peg_parser operator+(const char * str, const common_peg_parser & p) { return p.builder().literal(str) + p; } common_peg_parser operator+(const std::string & str, const common_peg_parser & p) { return operator+(str.c_str(), p); } common_peg_parser operator<<(const char * str, const common_peg_parser & p) { return p.builder().literal(str) << p; } common_peg_parser operator<<(const std::string & str, const common_peg_parser & p) { return operator<<(str.c_str(), p); } common_peg_parser operator|(const char * str, const common_peg_parser & p) { return p.builder().literal(str) | p; } common_peg_parser operator|(const std::string & str, const common_peg_parser & p) { return operator|(str.c_str(), p); } common_peg_parser_builder::common_peg_parser_builder() {} common_peg_parser_builder::common_peg_parser_builder(common_peg_tokens tokens) { arena_.tokens_ = std::move(tokens); } common_peg_parser common_peg_parser_builder::until_one_of(const std::vector & delimiters) { common_peg_until_parser p; for (const auto & d : delimiters) { p.branches.push_back({ literal(d).id() }); } p.matcher = build_until_matcher(arena_, p.branches, p.include); return add(std::move(p)); } common_peg_parser common_peg_parser_builder::until(const common_peg_parser & delimiter) { common_peg_until_parser p; p.branches = { { delimiter.id() } }; p.matcher = build_until_matcher(arena_, p.branches, p.include); return add(std::move(p)); } common_peg_parser common_peg_parser_builder::through(const common_peg_parser & content, const std::vector & branches, bool optional) { if (branches.empty()) { throw std::runtime_error("through requires at least one branch"); } common_peg_until_parser p; p.branches = branches; p.include = true; p.optional = optional; p.content = content.id(); p.matcher = build_until_matcher(arena_, p.branches, p.include); return add(std::move(p)); } common_peg_parser common_peg_parser_builder::token(const std::string & piece) { auto token = arena_.tokens_.token_id(piece); if (token == LLAMA_TOKEN_NULL) { return literal(piece); } return add(common_peg_token_parser{token, piece}); } common_peg_parser common_peg_parser_builder::token(const std::vector & pieces, bool negate) { if (!negate) { std::vector alts; for (const auto & piece : pieces) { alts.push_back(token(piece)); } return choice(alts); } common_peg_token_not_parser p; for (const auto & piece : pieces) { auto token = arena_.tokens_.token_id(piece); if (token == LLAMA_TOKEN_NULL) { throw std::invalid_argument("token is not registered with the builder: " + piece); } p.tokens.push_back(token); } return add(p); } common_peg_parser common_peg_parser_builder::sequence(const std::vector & parsers) { // Flatten nested sequences std::vector flattened; for (const auto & p : parsers) { const auto & parser = arena_.get(p); if (auto seq = std::get_if(&parser)) { flattened.insert(flattened.end(), seq->children.begin(), seq->children.end()); } else { flattened.push_back(p); } } return wrap(arena_.add_parser(common_peg_sequence_parser{flattened})); } common_peg_parser common_peg_parser_builder::sequence(const std::vector & parsers) { std::vector ids; ids.reserve(parsers.size()); for (const auto & p : parsers) { ids.push_back(p.id()); } return sequence(ids); } common_peg_parser common_peg_parser_builder::sequence(std::initializer_list parsers) { std::vector ids; ids.reserve(parsers.size()); for (const auto & p : parsers) { ids.push_back(p.id()); } return sequence(ids); } common_peg_parser common_peg_parser_builder::choice(const std::vector & parsers) { // Flatten nested choices std::vector flattened; for (const auto & p : parsers) { const auto & parser = arena_.get(p); if (auto choice = std::get_if(&parser)) { flattened.insert(flattened.end(), choice->children.begin(), choice->children.end()); } else { flattened.push_back(p); } } return wrap(arena_.add_parser(common_peg_choice_parser{flattened})); } common_peg_parser common_peg_parser_builder::choice(const std::vector & parsers) { std::vector ids; ids.reserve(parsers.size()); for (const auto & p : parsers) { ids.push_back(p.id()); } return choice(ids); } common_peg_parser common_peg_parser_builder::choice(std::initializer_list parsers) { std::vector ids; ids.reserve(parsers.size()); for (const auto & p : parsers) { ids.push_back(p.id()); } return choice(ids); } common_peg_parser common_peg_parser_builder::chars(const std::string & classes, int min, int max) { auto [ranges, negated] = parse_char_classes(classes); return wrap(arena_.add_parser(common_peg_chars_parser{classes, ranges, negated, min, max})); } common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, common_chat_schema_document_ptr doc, const common_chat_schema & node, bool raw) { return wrap(arena_.add_parser(common_peg_schema_parser{p.id(), name, std::move(doc), &node, raw})); } common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, const common_json & schema, bool raw) { auto doc = std::make_shared(common_chat_schema_from_json(schema)); return this->schema(p, name, doc, *doc->root, raw); } common_peg_parser common_peg_parser_builder::rule(const std::string & name, const common_peg_parser & p, bool trigger) { auto clean_name = gbnf_format_rule_name(name); auto rule_id = arena_.add_parser(common_peg_rule_parser{clean_name, p.id(), trigger}); arena_.add_rule(clean_name, rule_id); return ref(clean_name); } common_peg_parser common_peg_parser_builder::rule(const std::string & name, const std::function & builder_fn, bool trigger) { auto clean_name = gbnf_format_rule_name(name); if (arena_.has_rule(clean_name)) { return ref(clean_name); } // Create placeholder rule to allow recursive references auto placeholder = any(); // Temporary placeholder auto placeholder_rule_id = arena_.add_parser(common_peg_rule_parser{clean_name, placeholder.id(), trigger}); arena_.add_rule(clean_name, placeholder_rule_id); // Build the actual parser auto parser = builder_fn(); // Replace placeholder with actual rule auto rule_id = arena_.add_parser(common_peg_rule_parser{clean_name, parser.id(), trigger}); arena_.rules_[clean_name] = rule_id; return ref(clean_name); } common_peg_parser common_peg_parser_builder::trigger_rule(const std::string & name, const common_peg_parser & start, const common_peg_parser & rest) { common_trie scratch; if (insert_delimiter(scratch, arena_, start.id()).count(0)) { throw std::invalid_argument("trigger start must not match the empty string"); } auto clean_name = gbnf_format_rule_name(name); auto rule_id = arena_.add_parser(common_peg_rule_parser{clean_name, (start + rest).id(), true, start.id(), rest.id()}); arena_.add_rule(clean_name, rule_id); return ref(clean_name); } void common_peg_parser_builder::set_root(const common_peg_parser & p) { arena_.set_root(p.id()); } common_peg_arena common_peg_parser_builder::build() { arena_.resolve_refs(); return std::move(arena_); } // String primitives common_peg_parser common_peg_parser_builder::string_content(char delimiter) { return wrap(arena_.add_parser(common_peg_string_parser{delimiter})); } common_peg_parser common_peg_parser_builder::double_quoted_string() { return rule("double-quoted-string", [this]() { return sequence({literal("\""), string_content('"'), literal("\"")}); }); } common_peg_parser common_peg_parser_builder::single_quoted_string() { return rule("single-quoted-string", [this]() { return sequence({literal("'"), string_content('\''), literal("'")}); }); } common_peg_parser common_peg_parser_builder::quoted_string() { return rule("quoted-string", [this]() { return choice({double_quoted_string(), single_quoted_string()}); }); } // JSON parsers common_peg_parser common_peg_parser_builder::json_number() { return rule("json-number", [this]() { auto digit1_9 = chars("[1-9]", 1, 1); auto digits = chars("[0-9]"); auto int_part = choice({literal("0"), sequence({digit1_9, chars("[0-9]", 0, -1)})}); auto frac = sequence({literal("."), digits}); auto exp = sequence({choice({literal("e"), literal("E")}), optional(chars("[+-]", 1, 1)), digits}); // Negative lookahead: only commit the number when the next character can't extend it. // At EOF in partial mode, chars returns NEED_MORE → negate propagates NEED_MORE → number not committed. // This prevents premature commits of partial numbers (e.g. "3" when "3.14" is incoming). auto not_number_continuation = negate(chars("[0-9.eE+-]", 1, 1)); return sequence({ optional(literal("-")), int_part, optional(frac), optional(exp), not_number_continuation }); }); } common_peg_parser common_peg_parser_builder::json_string() { return rule("json-string", [this]() { return sequence({literal("\""), string_content('"'), literal("\"")}); }); } common_peg_parser common_peg_parser_builder::json_bool() { return rule("json-bool", [this]() { return choice({literal("true"), literal("false")}); }); } common_peg_parser common_peg_parser_builder::json_null() { return rule("json-null", [this]() { return literal("null"); }); } common_peg_parser common_peg_parser_builder::json_object() { return rule("json-object", [this]() { auto ws = space(); auto member = sequence({json_string(), ws, literal(":"), ws, json()}); auto members = sequence({member, zero_or_more(sequence({ws, literal(","), ws, member}))}); return sequence({ literal("{"), ws, choice({ literal("}"), sequence({members, ws, literal("}")}) }) }); }); } common_peg_parser common_peg_parser_builder::json_array() { return rule("json-array", [this]() { auto ws = space(); auto elements = sequence({json(), zero_or_more(sequence({ws, literal(","), ws, json()}))}); return sequence({ literal("["), ws, choice({ literal("]"), sequence({elements, ws, literal("]")}) }) }); }); } common_peg_parser common_peg_parser_builder::json() { return rule("json-value", [this]() { return choice({ json_object(), json_array(), json_string(), json_number(), json_bool(), json_null() }); }); } common_peg_parser common_peg_parser_builder::python_string() { return rule("python-string", [this]() { return choice({double_quoted_string(), single_quoted_string()}); }); } common_peg_parser common_peg_parser_builder::python_number() { return json_number(); } common_peg_parser common_peg_parser_builder::python_bool() { return rule("python-bool", [this]() { return choice({literal("True"), literal("False")}); }); } common_peg_parser common_peg_parser_builder::python_null() { return rule("python-none", [this]() { return literal("None"); }); } common_peg_parser common_peg_parser_builder::python_dict() { return rule("python-dict", [this]() { auto ws = space(); auto member = sequence({python_string(), ws, literal(":"), ws, python_value()}); auto members = sequence({member, zero_or_more(sequence({ws, literal(","), ws, member}))}); return sequence({ literal("{"), ws, choice({ literal("}"), sequence({members, ws, literal("}")}) }), ws }); }); } common_peg_parser common_peg_parser_builder::python_array() { return rule("python-array", [this]() { auto ws = space(); auto elements = sequence({python_value(), zero_or_more(sequence({literal(","), ws, python_value()}))}); return sequence({ literal("["), ws, choice({ literal("]"), sequence({elements, ws, literal("]")}) }), ws }); }); } common_peg_parser common_peg_parser_builder::python_value() { return rule("python-value", [this]() { return choice({ python_dict(), python_array(), python_string(), python_number(), python_bool(), python_null() }); }); } common_peg_parser common_peg_parser_builder::marker() { auto sharp_bracket_parser = literal("<") + until(">") + literal(">"); auto square_bracket_parser = literal("[") + until("]") + literal("]"); return choice({ sharp_bracket_parser, square_bracket_parser }); } common_peg_parser common_peg_parser_builder::json_member(const std::string & key, const common_peg_parser & p) { auto ws = space(); return sequence({ literal("\"" + key + "\""), ws, literal(":"), ws, p, }); } static std::string gbnf_escape_char_class(uint32_t c) { if (c == '-' || c == ']' || c == '[' || c == '\\') { return "\\" + std::string(1, (char) c); } // Escape whitespace control characters if (c == '\n') { return "\\n"; } if (c == '\t') { return "\\t"; } if (c == '\r') { return "\\r"; } // Printable ASCII if (c >= 0x20 && c <= 0x7E) { return std::string(1, (char) c); } // Hex escape char buf[16]; const char * hex = "0123456789ABCDEF"; if (c <= 0xFF) { buf[0] = '\\'; buf[1] = 'x'; buf[2] = hex[(c >> 4) & 0xF]; buf[3] = hex[c & 0xF]; buf[4] = '\0'; } else if (c <= 0xFFFF) { buf[0] = '\\'; buf[1] = 'u'; buf[2] = hex[(c >> 12) & 0xF]; buf[3] = hex[(c >> 8) & 0xF]; buf[4] = hex[(c >> 4) & 0xF]; buf[5] = hex[c & 0xF]; buf[6] = '\0'; } else { buf[0] = '\\'; buf[1] = 'U'; for (int i = 0; i < 8; i++) { buf[2 + i] = hex[(c >> ((7 - i) * 4)) & 0xF]; } buf[10] = '\0'; } return std::string(buf); } static std::string gbnf_char_class(const std::vector & chars, bool negate) { std::string s = negate ? "[^" : "["; for (uint32_t ch : chars) { s += gbnf_escape_char_class(ch); } return s + "]"; } // Token set such as <[1,5-9]>, with consecutive ids collapsed into ranges static std::string gbnf_token_set(const std::vector & tokens, bool negate) { std::string s = negate ? "!<[" : "<["; for (size_t i = 0; i < tokens.size();) { size_t j = i; while (j + 1 < tokens.size() && tokens[j + 1] == tokens[j] + 1) { j++; } s += (i == 0 ? "" : ",") + std::to_string(tokens[i]); if (j > i) { s += "-" + std::to_string(tokens[j]); } i = j + 1; } return s + "]>"; } // Symbols on the transitions out of an automaton state, codepoints and tokens kept apart struct gbnf_ac_symbols { std::vector chars; std::vector tokens; void add(const common_trie::symbol & sym) { (sym.is_token() ? tokens : chars).push_back(sym.value); } // One alternative for the codepoints and one for the tokens, each followed by next if given std::vector alternatives(const std::string & next = "") const { std::vector alts; const std::string tail = next.empty() ? "" : " " + next; if (!chars.empty()) { alts.push_back(gbnf_char_class(chars, false) + tail); } if (!tokens.empty()) { alts.push_back(gbnf_token_set(tokens, false) + tail); } return alts; } }; static std::string gbnf_ac_grammar( const common_grammar_builder & builder, const std::string & prefix, const common_trie & trie, const std::set & opaque, const std::function &, const std::map &, const std::string &, const std::function &)> & build_rule) { common_aho_corasick ac(trie); // With only tokens in the alphabet, the grammar can step a whole token at a time bool token_only = true; for (const auto & sym : ac.alphabet) { token_only = token_only && sym.is_token(); } auto state_name = [&](size_t s) -> std::string { if (s == 0) { return prefix; } std::string num = std::to_string(s); num = num.size() == 1 ? ("0" + num) : num; return prefix + "-" + num; }; for (size_t q = 0; q < ac.num_states(); q++) { if (ac.is_terminal(q)) { continue; // match states } std::map buckets; std::map completing_to; // completing symbols by the match state they reach gbnf_ac_symbols completing; // symbols that complete a delimiter gbnf_ac_symbols specific; // symbols with an explicit transition for (const auto & sym : ac.alphabet) { size_t d = ac.next(q, sym); if (ac.is_terminal(d)) { completing.add(sym); completing_to[d].add(sym); specific.add(sym); } else if (d != 0) { buckets[d].add(sym); // specific non-root destination specific.add(sym); } } // every other symbol goes back to the start std::string fallback; if (token_only) { fallback = gbnf_token_set(specific.tokens, true); } else { fallback = specific.chars.empty() ? "." : gbnf_char_class(specific.chars, true); // Char rules never match a token named by a token rule, so allow the other named tokens by name std::vector others; for (auto token : opaque) { if (std::find(specific.tokens.begin(), specific.tokens.end(), token) == specific.tokens.end()) { others.push_back(token); } } if (!others.empty()) { fallback = "(" + fallback + " | " + gbnf_token_set(others, false) + ")"; } } fallback += " " + state_name(0); builder.add_rule(state_name(q), build_rule(completing, completing_to, buckets, fallback, state_name)); } return state_name(0); } // GBNF grammar matching strings that contain no string in `strings` as a // substring. Emits the complement of an Aho-Corasick automaton DFA and returns // the start state rule name. // // ref: https://github.com/ggml-org/llama.cpp/pull/24839 static std::string gbnf_excluding_grammar(const common_grammar_builder & builder, const std::string & prefix, const common_trie & trie, const std::set & opaque) { return gbnf_ac_grammar(builder, prefix, trie, opaque, [](const gbnf_ac_symbols & /*completing*/, const std::map & /*completing_to*/, const std::map & buckets, const std::string & fallback, const std::function & state_name) { // every state accepts and completing symbols get no alternative, so a forbidden string never matches std::string rhs = "|"; for (const auto & [d, symbols] : buckets) { for (const auto & alt : symbols.alternatives(state_name(d))) { rhs += " " + alt + " |"; } } rhs += " " + fallback; return rhs; }); } // GBNF grammar matching up to the first pattern in the trie, then the rest rule of that pattern's value. // For the pattern "ab" with rest "x": // // prefix ::= [a] prefix-01 | [^a] prefix // prefix-01 ::= [b] rest | [a] prefix-01 | [^ab] prefix // rest ::= "x" static std::string gbnf_including_grammar(const common_grammar_builder & builder, const std::string & prefix, const common_trie & trie, const std::set & opaque, const std::vector & rests, bool optional) { common_aho_corasick ac(trie); // every pattern ending at a match state, the longest and the ones reached by suffix links auto rests_at = [&](size_t s, bool & terminal) { std::vector alts; terminal = false; for (; s != 0; s = ac.fail[s]) { auto pattern = ac.t.nodes[s].pattern; if (pattern < 0) { continue; } if (rests[pattern].empty()) { terminal = true; } else if (std::find(alts.begin(), alts.end(), rests[pattern]) == alts.end()) { alts.push_back(rests[pattern]); } } return alts; }; return gbnf_ac_grammar(builder, prefix, trie, opaque, [&](const gbnf_ac_symbols & /*completing*/, const std::map & completing_to, const std::map & buckets, const std::string & fallback, const std::function & state_name) { std::vector alts; gbnf_ac_symbols terminating; // symbols that complete a pattern with no rest for (const auto & [d, symbols] : completing_to) { bool terminal = false; auto rest = rests_at(d, terminal); if (!rest.empty()) { for (auto & alt : symbols.alternatives(rest.size() == 1 ? rest[0] : "(" + string_join(rest, " | ") + ")")) { alts.push_back(std::move(alt)); } } if (terminal) { terminating.chars.insert(terminating.chars.end(), symbols.chars.begin(), symbols.chars.end()); terminating.tokens.insert(terminating.tokens.end(), symbols.tokens.begin(), symbols.tokens.end()); } } for (auto & alt : terminating.alternatives()) { alts.push_back(std::move(alt)); } for (const auto & [d, symbols] : buckets) { for (auto & alt : symbols.alternatives(state_name(d))) { alts.push_back(std::move(alt)); } } // every other symbol keeps scanning from the start state alts.push_back(fallback); return (optional ? "| " : "") + string_join(alts, " | "); }); } // The rules and the tokens named by token rules that the grammar of a parser reaches struct common_peg_reachable { std::set rules; std::set tokens; }; static void collect_reachable( const common_peg_arena & arena, const common_peg_parser_id & root, common_peg_reachable & reachable ) { std::set visited; std::function visit = [&](common_peg_parser_id id) { const auto & parser = arena.get(id); std::visit([&](const auto & p) { using T = std::decay_t; if constexpr (std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v) { // These parsers do not have any children } else if constexpr (std::is_same_v) { reachable.tokens.insert(p.token); } else if constexpr (std::is_same_v) { reachable.tokens.insert(p.tokens.begin(), p.tokens.end()); } else if constexpr (std::is_same_v) { // The delimiters name their tokens in the grammar if (p.content != COMMON_PEG_INVALID_PARSER_ID) { visit(p.content); } for (const auto & branch : p.branches) { visit(branch.delimiter); if (branch.rest != COMMON_PEG_INVALID_PARSER_ID) { visit(branch.rest); } } } else if constexpr (std::is_same_v) { for (auto child : p.children) { visit(child); } } else if constexpr (std::is_same_v) { for (auto child : p.children) { visit(child); } } else if constexpr (std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v) { visit(p.child); } else if constexpr (std::is_same_v) { if (visited.find(p.name) == visited.end()) { visited.insert(p.name); reachable.rules.insert(p.name); visit(p.child); } } else if constexpr (std::is_same_v) { // Traverse rules so we pick up everything auto referenced_rule = arena.get_rule(p.name); visit(referenced_rule); } else { static_assert(is_always_false_v); } }, parser); }; visit(root); } // GBNF generation implementation void common_peg_arena::build_grammar(const common_grammar_builder & builder, bool lazy) const { common_peg_reachable reachable; // A raw string value is parsed by the child rather than constrained by the schema auto schema_delegates = [](const common_peg_schema_parser & s) -> bool { return !s.node || (s.raw && s.node->may_be_string()); }; // Unwrap the parser so we can properly check if it's a sequence or choice auto effective_parser = [&](common_peg_parser_id id) -> const common_peg_parser_variant & { while (true) { const auto & p = parsers_.at(id); if (const auto * tag = std::get_if(&p)) { id = tag->child; } else if (const auto * atomic = std::get_if(&p)) { id = atomic->child; } else if (const auto * schema = std::get_if(&p)) { if (schema_delegates(*schema)) { id = schema->child; } else { return p; } } else { return p; } } }; // Generate GBNF for a parser std::function to_gbnf = [&](common_peg_parser_id id) -> std::string { const auto & parser = parsers_.at(id); return std::visit([&](const auto & p) -> std::string { using T = std::decay_t; if constexpr (std::is_same_v || std::is_same_v || std::is_same_v) { return ""; } else if constexpr (std::is_same_v) { return gbnf_format_literal(p.literal); } else if constexpr (std::is_same_v) { return "<[" + std::to_string(p.token) + "]>"; } else if constexpr (std::is_same_v) { return gbnf_token_set(std::vector(p.tokens.begin(), p.tokens.end()), true); } else if constexpr (std::is_same_v) { std::string s; for (const auto & child : p.children) { auto child_gbnf = to_gbnf(child); if (child_gbnf.empty()) { continue; } if (!s.empty()) { s += " "; } const auto & child_parser = effective_parser(child); if (std::holds_alternative(child_parser) || std::holds_alternative(child_parser)) { s += "(" + child_gbnf + ")"; } else { s += child_gbnf; } } return s; } else if constexpr (std::is_same_v) { std::string s; for (const auto & child : p.children) { if (!s.empty()) { s += " | "; } auto child_gbnf = to_gbnf(child); const auto & child_parser = effective_parser(child); if (std::holds_alternative(child_parser)) { s += "(" + child_gbnf + ")"; } else { s += child_gbnf; } } return s; } else if constexpr (std::is_same_v) { auto child_gbnf = to_gbnf(p.child); const auto & child_parser = effective_parser(p.child); if (std::holds_alternative(child_parser) || std::holds_alternative(child_parser)) { child_gbnf = "(" + child_gbnf + ")"; } if (p.min_count == 0 && p.max_count == 1) { return child_gbnf + "?"; } if (p.min_count == 0 && p.max_count == -1) { return child_gbnf + "*"; } if (p.min_count == 1 && p.max_count == -1) { return child_gbnf + "+"; } if (p.max_count == -1) { return child_gbnf + "{" + std::to_string(p.min_count) + ",}"; } if (p.min_count == p.max_count) { if (p.min_count == 1) { return child_gbnf; } return child_gbnf + "{" + std::to_string(p.min_count) + "}"; } return child_gbnf + "{" + std::to_string(p.min_count) + "," + std::to_string(p.max_count) + "}"; } else if constexpr (std::is_same_v || std::is_same_v) { return ""; // Lookahead not supported in GBNF } else if constexpr (std::is_same_v) { return "."; } else if constexpr (std::is_same_v) { return "space"; } else if constexpr (std::is_same_v) { std::string result = p.pattern; if (p.min_count == 0 && p.max_count == 1) { return result + "?"; } if (p.min_count == 0 && p.max_count == -1) { return result + "*"; } if (p.min_count == 1 && p.max_count == -1) { return result + "+"; } if (p.max_count == -1) { return result + "{" + std::to_string(p.min_count) + ",}"; } if (p.min_count == p.max_count) { if (p.min_count == 1) { return result; } return result + "{" + std::to_string(p.min_count) + "}"; } return result + "{" + std::to_string(p.min_count) + "," + std::to_string(p.max_count) + "}"; } else if constexpr (std::is_same_v) { const std::string delim(1, p.delimiter); return R"(( [^)" + delim + R"(\\] | "\\" ( [)" + delim + R"(\\/ bfnrt] | "u" [0-9a-fA-F]{4} ) )*)"; } else if constexpr (std::is_same_v) { auto prefix = "until-" + std::to_string(id); if (!p.include) { if (p.branches.empty()) { return ".*"; } return gbnf_excluding_grammar(builder, prefix, p.matcher, reachable.tokens); } std::vector branch_rests; for (size_t b = 0; b < p.branches.size(); b++) { auto rest = p.branches[b].rest; branch_rests.push_back(rest == COMMON_PEG_INVALID_PARSER_ID ? "" : builder.add_rule(prefix + "-rest-" + std::to_string(b), to_gbnf(rest))); } return gbnf_including_grammar(builder, prefix, p.matcher, reachable.tokens, branch_rests, p.optional); } else if constexpr (std::is_same_v) { if (schema_delegates(p)) { return to_gbnf(p.child); } return builder.add_schema(p.name, *p.node); } else if constexpr (std::is_same_v) { return p.name; } else if constexpr (std::is_same_v) { // Refs should not exist after flattening, but kept just in case return p.name; } else if constexpr (std::is_same_v) { return to_gbnf(p.child); } else if constexpr (std::is_same_v) { return to_gbnf(p.child); } else if constexpr (std::is_same_v) { return p.grammar; } else { static_assert(is_always_false_v); } }, parser); }; // Trigger rules, sorted for a predictable order std::vector trigger_rules; for (const auto & [name, id] : rules_) { if (auto rule = std::get_if(&parsers_.at(id))) { if (rule->trigger) { trigger_rules.push_back(rule); } } } std::sort(trigger_rules.begin(), trigger_rules.end(), [](const auto * a, const auto * b) { return a->name < b->name; }); // A lazy grammar scans for the starts when the trigger rules were built from a start and rest bool scans = false; for (const auto * rule : trigger_rules) { scans = scans || rule->start != COMMON_PEG_INVALID_PARSER_ID; } if (scans) { for (const auto * rule : trigger_rules) { if (rule->start == COMMON_PEG_INVALID_PARSER_ID) { throw std::runtime_error("trigger rule without a start in a scanning grammar: " + rule->name); } } } if (lazy && scans) { for (const auto * rule : trigger_rules) { collect_reachable(*this, rule->start, reachable); collect_reachable(*this, rule->rest, reachable); } } else if (lazy) { for (const auto * rule : trigger_rules) { collect_reachable(*this, rules_.at(rule->name), reachable); } } else { collect_reachable(*this, root_, reachable); } // Create GBNF rules for all reachable rules for (const auto & [name, rule_id] : rules_) { if (reachable.rules.find(name) == reachable.rules.end()) { continue; } const auto & parser = parsers_.at(rule_id); if (auto rule = std::get_if(&parser)) { builder.add_rule(rule->name, to_gbnf(rule->child)); } } if (lazy && scans) { // The root scans for a trigger start and then constrains its rest common_trie starts; std::vector rests; for (const auto * rule : trigger_rules) { auto value = (int32_t) rests.size(); rests.push_back(builder.add_rule(rule->name + "-rest", to_gbnf(rule->rest))); for (size_t end : insert_delimiter(starts, *this, rule->start)) { auto owner = starts.mark(end, value); if (owner != value) { throw std::runtime_error("trigger rules " + trigger_rules[owner]->name + " and " + rule->name + " must not share a start"); } } } builder.add_rule("root", gbnf_including_grammar(builder, "trigger", starts, reachable.tokens, rests, /* optional = */ true)); } else if (lazy) { // Generate root rule from trigger rules only std::vector trigger_names; for (const auto * rule : trigger_rules) { trigger_names.push_back(rule->name); } builder.add_rule("root", string_join(trigger_names, " | ")); } else if (root_ != COMMON_PEG_INVALID_PARSER_ID) { builder.add_rule("root", to_gbnf(root_)); } } common_peg_arena build_peg_parser(const std::function & fn) { common_peg_parser_builder builder; builder.set_root(fn(builder)); return builder.build(); }