#include "tests.h" #include "json-schema-to-grammar.h" #include static std::string trim_leading_space(const std::string & s) { static const std::regex leading_ws_re = std::regex(R"((^|\n)\s+)"); return std::regex_replace(s, leading_ws_re, "$1"); } static void assert_gbnf_equal(testing & t, const std::string & expected, const std::string & actual) { t.assert_equal("gbnf are equal", trim_leading_space(expected), trim_leading_space(actual)); } void test_gbnf_generation(testing &t) { t.test("literal grammar generation", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello"); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= "hello" space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); t.test("char class grammar", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[a-z]", 1, 1); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= [a-z] space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); t.test("sequence grammar", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello") + p.literal(" ") + p.literal("world"); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= "hello" " " "world" space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); t.test("choice grammar", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("cat") | p.literal("dog"); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= "cat" | "dog" space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); t.test("one_or_more grammar", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("a")); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= "a"+ space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); t.test("zero_or_more grammar", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("a")); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= "a"* space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); t.test("optional grammar", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= "hello" " world"? space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); t.test("until grammar", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.until(""); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= until-1 space ::= | " " | "\n"{1,2} [ \t]{0,20} until-1 ::= | [<] until-1-01 | [^<] until-1 until-1-01 ::= | [<] until-1-01 | [/] until-1-02 | [^/<] until-1 until-1-02 ::= | [<] until-1-01 | [t] until-1-03 | [^] until-1 )""", gbnf); }); t.test("until grammar overlapping delimiter", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.until("\n\n"); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= until-1 space ::= | " " | "\n"{1,2} [ \t]{0,20} until-1 ::= | [\n] until-1-01 | [^\n] until-1 until-1-01 ::= | [\n] until-1-01 | [<] until-1-02 | [^\n<] until-1 until-1-02 ::= | [\n] until-1-01 | [/] until-1-03 | [^\n/] until-1 until-1-03 ::= | [\n] until-1-01 | [p] until-1-04 | [^\np] until-1 until-1-04 ::= | [\n] until-1-01 | [a] until-1-05 | [^\na] until-1 until-1-05 ::= | [\n] until-1-01 | [r] until-1-06 | [^\nr] until-1 until-1-06 ::= | [\n] until-1-01 | [a] until-1-07 | [^\na] until-1 until-1-07 ::= | [\n] until-1-01 | [m] until-1-08 | [^\nm] until-1 until-1-08 ::= | [\n] until-1-01 | [e] until-1-09 | [^\ne] until-1 until-1-09 ::= | [\n] until-1-01 | [t] until-1-10 | [^\nt] until-1 until-1-10 ::= | [\n] until-1-01 | [e] until-1-11 | [^\ne] until-1 until-1-11 ::= | [\n] until-1-01 | [r] until-1-12 | [^\nr] until-1 until-1-12 ::= | [\n] until-1-01 | [>] until-1-13 | [^\n>] until-1 until-1-13 ::= | [^\n] until-1 )""", gbnf); }); // DeepSeek-V3.2 tag prefix. The DSML token (|DSML|) embeds U+FF5C, // so the delimiter mixes ASCII and multi-byte codepoints. t.test("until grammar unicode delimiter", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.until("<|DSML|"); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= until-1 space ::= | " " | "\n"{1,2} [ \t]{0,20} until-1 ::= | [<] until-1-01 | [^<] until-1 until-1-01 ::= | [<] until-1-01 | [\uFF5C] until-1-02 | [^<\uFF5C] until-1 until-1-02 ::= | [<] until-1-01 | [D] until-1-03 | [^"), ""); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= until-3 space ::= | " " | "\n"{1,2} [ \t]{0,20} until-3 ::= [<] until-3-01 | [^<] until-3 until-3-01 ::= [<] until-3-01 | [/] until-3-02 | [^/<] until-3 until-3-02 ::= [<] until-3-01 | [t] until-3-03 | [^] | [<] until-3-01 | [^<>] until-3 )""", gbnf); }); t.test("through grammar terminates at first delimiter", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.through(p.until("\n\n"), "\n\n"); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= until-3 space ::= | " " | "\n"{1,2} [ \t]{0,20} until-3 ::= [\n] until-3-01 | [^\n] until-3 until-3-01 ::= [\n] until-3-01 | [<] until-3-02 | [^\n<] until-3 until-3-02 ::= [\n] until-3-01 | [/] until-3-03 | [^\n/] until-3 until-3-03 ::= [\n] until-3-01 | [p] until-3-04 | [^\np] until-3 until-3-04 ::= [\n] until-3-01 | [a] until-3-05 | [^\na] until-3 until-3-05 ::= [\n] until-3-01 | [r] until-3-06 | [^\nr] until-3 until-3-06 ::= [\n] until-3-01 | [a] until-3-07 | [^\na] until-3 until-3-07 ::= [\n] until-3-01 | [m] until-3-08 | [^\nm] until-3 until-3-08 ::= [\n] until-3-01 | [e] until-3-09 | [^\ne] until-3 until-3-09 ::= [\n] until-3-01 | [t] until-3-10 | [^\nt] until-3 until-3-10 ::= [\n] until-3-01 | [e] until-3-11 | [^\ne] until-3 until-3-11 ::= [\n] until-3-01 | [r] until-3-12 | [^\nr] until-3 until-3-12 ::= [\n] until-3-01 | [>] until-3-13 | [^\n>] until-3 until-3-13 ::= [\n] | [^\n] until-3 )""", gbnf); }); t.test("through grammar multiple delimiters", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.through(p.eps(), { { p.literal("ab") }, { p.literal("cd") }, { p.literal("ef") } }); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= until-4 space ::= | " " | "\n"{1,2} [ \t]{0,20} until-4 ::= [a] until-4-01 | [c] until-4-03 | [e] until-4-05 | [^ace] until-4 until-4-01 ::= [b] | [a] until-4-01 | [c] until-4-03 | [e] until-4-05 | [^abce] until-4 until-4-03 ::= [d] | [a] until-4-01 | [c] until-4-03 | [e] until-4-05 | [^acde] until-4 until-4-05 ::= [f] | [a] until-4-01 | [c] until-4-03 | [e] until-4-05 | [^acef] until-4 )""", gbnf); }); t.test("until grammar with parser delimiters", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.until((p.literal("a") + p.literal("b")) | p.literal("cd")); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= until-5 space ::= | " " | "\n"{1,2} [ \t]{0,20} until-5 ::= | [a] until-5-01 | [c] until-5-03 | [^ac] until-5 until-5-01 ::= | [a] until-5-01 | [c] until-5-03 | [^abc] until-5 until-5-03 ::= | [a] until-5-01 | [c] until-5-03 | [^acd] until-5 )""", gbnf); }); t.test("delimiters see through tags and atomics", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.until(p.tag("open", p.literal("a") + p.atomic(p.literal("b")))); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= until-5 space ::= | " " | "\n"{1,2} [ \t]{0,20} until-5 ::= | [a] until-5-01 | [^a] until-5 until-5-01 ::= | [a] until-5-01 | [^ab] until-5 )""", gbnf); }); t.test("delimiters expand a choice in a sequence", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.until(p.literal("a") + (p.literal("b") | p.literal("d"))); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= until-5 space ::= | " " | "\n"{1,2} [ \t]{0,20} until-5 ::= | [a] until-5-01 | [^a] until-5 until-5-01 ::= | [a] until-5-01 | [^abd] until-5 )""", gbnf); }); t.test("delimiters reject anything but sequences, choices, literals, and tokens", [](testing &t) { bool threw = false; try { build_peg_parser([](common_peg_parser_builder & p) { return p.until(p.literal("a") + p.space()); }); } catch (const std::invalid_argument &) { threw = true; } t.assert_equal("space_rejected", true, threw); }); t.test("through branches reject a shared delimiter", [](testing &t) { bool threw = false; try { build_peg_parser([](common_peg_parser_builder & p) { return p.through(p.eps(), { { p.literal("ab"), p.literal("x") }, { p.literal("ab"), p.literal("y") } }); }); } catch (const std::invalid_argument &) { threw = true; } t.assert_equal("duplicate_rejected", true, threw); }); t.test("through grammar branches continue with their rest", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.through(p.eps(), { { p.literal("ab"), p.literal("x") }, { p.literal("cd") } }); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= until-4 space ::= | " " | "\n"{1,2} [ \t]{0,20} until-4 ::= [a] until-4-01 | [c] until-4-03 | [^ac] until-4 until-4-01 ::= [b] until-4-rest-0 | [a] until-4-01 | [c] until-4-03 | [^abc] until-4 until-4-03 ::= [d] | [a] until-4-01 | [c] until-4-03 | [^acd] until-4 until-4-rest-0 ::= "x" )""", gbnf); }); t.test("through grammar optional may end before a delimiter", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.through(p.eps(), { { p.literal("ab"), p.literal("x") } }, true); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= until-3 space ::= | " " | "\n"{1,2} [ \t]{0,20} until-3 ::= | [a] until-3-01 | [^a] until-3 until-3-01 ::= | [b] until-3-rest-0 | [a] until-3-01 | [^ab] until-3 until-3-rest-0 ::= "x" )""", gbnf); }); t.test("trigger rules emit their start and rest when not lazy", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { auto ab = p.literal("ab"); auto cd = p.literal("cd"); auto scan = p.until(ab | cd); return scan + (p.trigger_rule("one", ab, p.literal("x")) | p.trigger_rule("two", cd, p.literal("y"))); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder, false); }); assert_gbnf_equal(t, R"""( one ::= "ab" "x" root ::= until-3 (one | two) space ::= | " " | "\n"{1,2} [ \t]{0,20} two ::= "cd" "y" until-3 ::= | [a] until-3-01 | [c] until-3-03 | [^ac] until-3 until-3-01 ::= | [a] until-3-01 | [c] until-3-03 | [^abc] until-3 until-3-03 ::= | [a] until-3-01 | [c] until-3-03 | [^acd] until-3 )""", gbnf); }); t.test("trigger rules make the lazy grammar scan for their starts", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { auto ab = p.literal("ab"); auto cd = p.literal("cd"); auto scan = p.until(ab | cd); return scan + (p.trigger_rule("one", ab, p.literal("x")) | p.trigger_rule("two", cd, p.literal("y"))); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder, true); }); assert_gbnf_equal(t, R"""( one-rest ::= "x" root ::= trigger space ::= | " " | "\n"{1,2} [ \t]{0,20} trigger ::= | [a] trigger-01 | [c] trigger-03 | [^ac] trigger trigger-01 ::= | [b] one-rest | [a] trigger-01 | [c] trigger-03 | [^abc] trigger trigger-03 ::= | [d] two-rest | [a] trigger-01 | [c] trigger-03 | [^acd] trigger two-rest ::= "y" )""", gbnf); }); t.test("scanning grammar lets other named tokens through", [](testing &t) { common_peg_tokens tokens; tokens.ids.emplace("", 1); tokens.ids.emplace("", 2); tokens.tokens.emplace(1, common_peg_token{ "", LLAMA_TOKEN_ATTR_CONTROL }); tokens.tokens.emplace(2, common_peg_token{ "", LLAMA_TOKEN_ATTR_CONTROL }); common_peg_parser_builder p(tokens); auto start = p.token("") + p.literal("x"); p.set_root(p.until(start) + p.trigger_rule("tool", start, p.token(""))); auto parser = p.build(); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder, true); }); assert_gbnf_equal(t, R"""( root ::= trigger space ::= | " " | "\n"{1,2} [ \t]{0,20} tool-rest ::= <[2]> trigger ::= | <[1]> trigger-01 | (. | <[2]>) trigger trigger-01 ::= | [x] tool-rest | <[1]> trigger-01 | ([^x] | <[2]>) trigger )""", gbnf); }); t.test("token sets", [](testing &t) { common_peg_tokens tokens; tokens.ids.emplace("", 1); tokens.ids.emplace("", 2); tokens.tokens.emplace(1, common_peg_token{ "", LLAMA_TOKEN_ATTR_CONTROL }); tokens.tokens.emplace(2, common_peg_token{ "", LLAMA_TOKEN_ATTR_CONTROL }); common_peg_parser_builder p(tokens); p.set_root(p.token({ "", "" }) + p.token({ "", "" }, true)); auto parser = p.build(); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= (<[1]> | <[2]>) !<[1-2]> space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); t.test("through grammar suffix delimiter offers both rests", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.through(p.eps(), { { p.literal("xab"), p.literal("1") }, { p.literal("ab"), p.literal("2") } }); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= until-5 space ::= | " " | "\n"{1,2} [ \t]{0,20} until-5 ::= [x] until-5-01 | [a] until-5-04 | [^ax] until-5 until-5-01 ::= [x] until-5-01 | [a] until-5-02 | [^ax] until-5 until-5-02 ::= [b] (until-5-rest-0 | until-5-rest-1) | [x] until-5-01 | [a] until-5-04 | [^abx] until-5 until-5-04 ::= [b] until-5-rest-1 | [x] until-5-01 | [a] until-5-04 | [^abx] until-5 until-5-rest-0 ::= "1" until-5-rest-1 ::= "2" )""", gbnf); }); t.test("complex expressions with parentheses", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("a") | p.literal("b")); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= ("a" | "b")+ space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); t.test("rule references", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { auto digit = p.rule("digit", p.chars("[0-9]", 1, 1)); return p.one_or_more(digit); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( digit ::= [0-9] root ::= digit+ space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); t.test("escaping in literals", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello\nworld\n!"); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= "hello\nworld\n!" space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); t.test("operator<< (whitespace insertion)", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello") << p.literal("world"); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= "hello" space "world" space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); t.test("emit only reachable rules", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { p.rule("orphan", p.literal("orphan")); return p.literal("hello") + p.rule("child", p.literal(" world")); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( child ::= " world" root ::= "hello" child space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); t.test("tagged choice inside sequence gets parenthesized", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("a") + p.tag("t", p.literal("b") | p.literal("c")); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= "a" ("b" | "c") space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); t.test("tagged sequence inside choice gets parenthesized", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.tag("t", p.literal("a") + p.literal("b")) | p.literal("c"); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= "a" "b" | "c" space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); t.test("atomic choice inside repetition gets parenthesized", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.atomic(p.literal("a") | p.literal("b"))); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= ("a" | "b")+ space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); t.test("silent parser emits nothing in gbnf", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello") + p.gbnf(p.literal("world"), ""); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= "hello" space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); t.test("silent choice inside sequence emits nothing", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("a") + p.gbnf(p.literal("b") | p.literal("c"), "") + p.literal("d"); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= "a" "d" space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); t.test("silent wrapped in tag emits nothing", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("a") + p.tag("t", p.gbnf(p.literal("b"), "")); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= "a" space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); t.test("gbnf parser emits custom grammar", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("a") + p.gbnf(p.literal("b"), "[a-z]+"); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= "a" [a-z]+ space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); t.test("nested transparent wrappers get parenthesized", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("x") + p.tag("outer", p.atomic(p.literal("a") | p.literal("b"))); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= "x" ("a" | "b") space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); t.test("emit only trigger rules (and references)", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { auto rule1 = p.rule("rule-1", p.literal("a") + p.ref("rule-2")); p.rule("rule-2", p.literal("b") + p.ref("rule-3"), true); p.rule("rule-3", p.literal("c") + p.ref("rule-4")); p.rule("rule-4", p.literal("d"), true); return rule1; }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); assert_gbnf_equal(t, R"""( root ::= rule-1 rule-1 ::= "a" rule-2 rule-2 ::= "b" rule-3 rule-3 ::= "c" rule-4 rule-4 ::= "d" space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); auto gbnf_lazy = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder, true); }); assert_gbnf_equal(t, R"""( root ::= rule-2 | rule-4 rule-2 ::= "b" rule-3 rule-3 ::= "c" rule-4 rule-4 ::= "d" space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf_lazy); }); }