chat : implement token aware parsing (#29307)
This commit is contained in:
1 parent
0b51bb2fed
commit
1014775c8a
30 files changed
+1988
-502
No files matched your search
@@ -179,6 +179,14 @@ int main()
|
||||
verify_failure(deep_grammar.c_str());
|
||||
}
|
||||
|
||||
verify_failure(R"""(
|
||||
root ::= <[9-5]>
|
||||
)""");
|
||||
|
||||
verify_failure(R"""(
|
||||
root ::= <[4294967296]>
|
||||
)""");
|
||||
|
||||
verify_parsing(R"""(
|
||||
root ::= "a"{0,5000}
|
||||
)""", {
|
||||
@@ -594,5 +602,21 @@ int main()
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
});
|
||||
|
||||
verify_parsing(R"""(
|
||||
root ::= <[1,5-9,12]> !<[11,20-29]>
|
||||
)""", {
|
||||
{"root", 0}
|
||||
}, {
|
||||
// root (index 0)
|
||||
{LLAMA_GRETYPE_TOKEN, 1},
|
||||
{LLAMA_GRETYPE_TOKEN_ALT, 5},
|
||||
{LLAMA_GRETYPE_TOKEN_RNG_UPPER, 9},
|
||||
{LLAMA_GRETYPE_TOKEN_ALT, 12},
|
||||
{LLAMA_GRETYPE_TOKEN_NOT, 11},
|
||||
{LLAMA_GRETYPE_TOKEN_ALT, 20},
|
||||
{LLAMA_GRETYPE_TOKEN_RNG_UPPER, 29},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
});
|
||||
|
||||
return 0;
|
||||
}
|
||||
Reference in new issue
Block a user