chat : implement token aware parsing (#29307)

This commit is contained in:
Aldehir Rojas authored and GitHub committed 2026-11-22 11:36:03 -06:00
1 parent 0b51bb2fed
commit 1014775c8a
30 files changed
+1988 -502

No files matched your search

+24
View File
@@ -179,6 +179,14 @@ int main()
verify_failure(deep_grammar.c_str());
}
verify_failure(R"""(
root ::= <[9-5]>
)""");
verify_failure(R"""(
root ::= <[4294967296]>
)""");
verify_parsing(R"""(
root ::= "a"{0,5000}
)""", {
@@ -594,5 +602,21 @@ int main()
{LLAMA_GRETYPE_END, 0},
});
verify_parsing(R"""(
root ::= <[1,5-9,12]> !<[11,20-29]>
)""", {
{"root", 0}
}, {
// root (index 0)
{LLAMA_GRETYPE_TOKEN, 1},
{LLAMA_GRETYPE_TOKEN_ALT, 5},
{LLAMA_GRETYPE_TOKEN_RNG_UPPER, 9},
{LLAMA_GRETYPE_TOKEN_ALT, 12},
{LLAMA_GRETYPE_TOKEN_NOT, 11},
{LLAMA_GRETYPE_TOKEN_ALT, 20},
{LLAMA_GRETYPE_TOKEN_RNG_UPPER, 29},
{LLAMA_GRETYPE_END, 0},
});
return 0;
}