chat : implement token aware parsing (#29307)

This commit is contained in:
Aldehir Rojas authored and GitHub committed 2026-11-22 11:36:03 -06:00
1 parent 0b51bb2fed
commit 1014775c8a
30 files changed
+1988 -502

No files matched your search

+6 -4
View File
@@ -373,7 +373,9 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
auto & p = ctx.p;
const auto & inputs = ctx.inputs;
auto until_suffix = p.rule("until-suffix", p.until(arguments.value_suffix));
// A raw string value needs a suffix to end on, without one it is parsed as JSON
bool raw_strings = !arguments.value_suffix.empty();
auto until_suffix = raw_strings ? p.rule("until-suffix", p.until(arguments.value_suffix)) : p.eps();
common_peg_parser tool_choice = p.choice();
@@ -389,9 +391,9 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
p.tool_arg(p.tool_arg_open(arguments.name_prefix + p.tool_arg_name(p.literal(param.name)) +
arguments.name_suffix) +
arguments.value_prefix +
(param.schema->may_be_string() ?
p.ac(p.tool_arg_string_value(until_suffix) +
p.tool_arg_close(p.literal(arguments.value_suffix)), arguments.value_suffix) :
(raw_strings && param.schema->may_be_string() ?
p.through(p.tool_arg_string_value(until_suffix),
p.tool_arg_close(p.literal(arguments.value_suffix))) :
(p.tool_arg_json_value(p.schema(
p.json(), "tool-" + std::to_string(tool_index) + "-arg-" + std::to_string(param_index) + "-schema", doc, *param.schema)) +
p.tool_arg_close(p.literal(arguments.value_suffix)))));