chat : trigger lazy grammar on bare JSON tool calls (#32497)
Models like Llama 3.x emit tool calls as plain {"name": ...} JSON without a start marker, so with tool_choice=auto no grammar was produced at all. Use a pattern trigger on the opening of the call object instead.
Assisted-by: Claude Code
This commit is contained in:
1 parent
9c10c94e16
commit
42f6a58fd1
2 files changed
+41
-4
No files matched your search
@@ -69,9 +69,21 @@ common_chat_params peg_generator::generate_parser(const common_chat_template &
|
||||
std::string trigger_marker = !autoparser.tools.format.section_start.empty() ? autoparser.tools.format.section_start :
|
||||
autoparser.tools.format.per_call_start;
|
||||
|
||||
// bare JSON calls without a start marker (e.g. Llama 3.x): trigger the lazy grammar on the call's opening instead
|
||||
std::string trigger_pattern;
|
||||
{
|
||||
const auto & fmt = autoparser.tools.format;
|
||||
bool top_level_name = (fmt.function_field.empty() || fmt.function_field == "function") &&
|
||||
fmt.name_field.find('.') == std::string::npos;
|
||||
if (trigger_marker.empty() && fmt.mode == tool_format::JSON_NATIVE && !fmt.fun_name_is_key &&
|
||||
!fmt.tools_array_wrapped && top_level_name) {
|
||||
trigger_pattern = "\\{\\s*\"" + regex_escape(fmt.name_field) + "\"\\s*:";
|
||||
}
|
||||
}
|
||||
|
||||
bool has_response_format = !inputs.json_schema.empty() && inputs.json_schema.is_object();
|
||||
bool include_grammar = has_response_format || (has_tools &&
|
||||
((inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO && !trigger_marker.empty()) ||
|
||||
((inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO && (!trigger_marker.empty() || !trigger_pattern.empty())) ||
|
||||
inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
||||
|
||||
if (include_grammar) {
|
||||
@@ -82,9 +94,11 @@ common_chat_params peg_generator::generate_parser(const common_chat_template &
|
||||
|
||||
// Set grammar triggers based on tool section markers (fall back to per-call markers)
|
||||
if (data.grammar_lazy) {
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, trigger_marker }
|
||||
};
|
||||
if (!trigger_marker.empty()) {
|
||||
data.grammar_triggers = { { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, trigger_marker } };
|
||||
} else {
|
||||
data.grammar_triggers = { { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, trigger_pattern } };
|
||||
}
|
||||
if (autoparser.tools.format.openai_wrapper_trigger) {
|
||||
// model emits the OpenAI function wrapper, trigger on it
|
||||
data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "{\"type\": \"function\"," });
|
||||
|
||||
Reference in new issue
Block a user