chat : implement token aware parsing (#29307)
This commit is contained in:
1 parent
0b51bb2fed
commit
1014775c8a
30 files changed
+1988
-502
No files matched your search
@@ -373,7 +373,9 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
|
||||
auto & p = ctx.p;
|
||||
const auto & inputs = ctx.inputs;
|
||||
|
||||
auto until_suffix = p.rule("until-suffix", p.until(arguments.value_suffix));
|
||||
// A raw string value needs a suffix to end on, without one it is parsed as JSON
|
||||
bool raw_strings = !arguments.value_suffix.empty();
|
||||
auto until_suffix = raw_strings ? p.rule("until-suffix", p.until(arguments.value_suffix)) : p.eps();
|
||||
|
||||
common_peg_parser tool_choice = p.choice();
|
||||
|
||||
@@ -389,9 +391,9 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
|
||||
p.tool_arg(p.tool_arg_open(arguments.name_prefix + p.tool_arg_name(p.literal(param.name)) +
|
||||
arguments.name_suffix) +
|
||||
arguments.value_prefix +
|
||||
(param.schema->may_be_string() ?
|
||||
p.ac(p.tool_arg_string_value(until_suffix) +
|
||||
p.tool_arg_close(p.literal(arguments.value_suffix)), arguments.value_suffix) :
|
||||
(raw_strings && param.schema->may_be_string() ?
|
||||
p.through(p.tool_arg_string_value(until_suffix),
|
||||
p.tool_arg_close(p.literal(arguments.value_suffix))) :
|
||||
(p.tool_arg_json_value(p.schema(
|
||||
p.json(), "tool-" + std::to_string(tool_index) + "-arg-" + std::to_string(param_index) + "-schema", doc, *param.schema)) +
|
||||
p.tool_arg_close(p.literal(arguments.value_suffix)))));
|
||||
|
||||
Reference in new issue
Block a user