chat : refactor gemma4 to build loose schema rules and clean up (#31915)

* chat : build a schema-shaped PEG parser for Gemma 4 tool arguments

* chat : enforce only the key structure of Gemma 4 tool arguments

* chat : remove old gemma4 workarounds

* cont : add node.as<>() to simplify static casts

* cont : add depth_guard and use numerical rule names

* cont : clean up

* cont : extract gemma 4 parsers

* cont : clean up

* cont : clean up

* cont : remove dynamic cast
This commit is contained in:
Aldehir Rojas authored and GitHub committed 2026-11-12 09:44:42 +02:00
1 parent 4fffab623e
commit 93cf736a8d
8 files changed
+426 -517

No files matched your search

+55
View File
@@ -3267,6 +3267,61 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
.expect(message_with_tool_calls("amount", R"({"orig": 1.5e10})"))
.run();
// The argument grammar follows the shape of the parameter schema: keys in
// alphabetical order, optional keys skipped in place, nested objects and arrays
// shaped the same way, any other value accepted as written.
common_chat_tool shape_tool{
/* .name = */ "shape",
/* .description = */ "Schema-shaped arguments",
/* .parameters = */ R"({
"type": "object",
"properties": {
"path": {"type": "string"},
"edits": {
"type": "array",
"items": {
"type": "object",
"properties": {
"oldText": {"type": "string"},
"newText": {"type": "string"}
},
"required": ["oldText", "newText"]
}
},
"mode": {"type": "string", "enum": ["fast", "safe"]},
"count": {"type": "integer"},
"headers": {
"type": "object",
"properties": {
"content-type": {"type": "string"},
"accept:lang": {"type": "string"}
}
},
"options": {"type": "object"}
},
"required": ["path", "edits"]
})",
};
tst.test(
"<|tool_call>call:shape{edits:[{newText:<|\"|>b<|\"|>,oldText:<|\"|>a<|\"|>}],path:<|\"|>f.c<|\"|>}<tool_call|>")
.tools({ shape_tool })
.expect(message_with_tool_calls("shape", R"({"edits":[{"newText":"b","oldText":"a"}],"path":"f.c"})"))
.run();
// Optional keys in place, quoted keys that are not identifiers
tst.test(
"<|tool_call>call:shape{count:3,edits:[],headers:{<|\"|>accept:lang<|\"|>:<|\"|>en<|\"|>,'content-type':<|\"|>text<|\"|>},mode:<|\"|>fast<|\"|>,options:{b:1,a:[true]},path:<|\"|>f<|\"|>}<tool_call|>")
.tools({ shape_tool })
.expect(message_with_tool_calls("shape", R"({"count":3,"edits":[],"headers":{"accept:lang":"en","content-type":"text"},"mode":"fast","options":{"b":1,"a":[true]},"path":"f"})"))
.run();
tst.test(
"<|tool_call>call:shape{edits:[],headers:{\"content-type\":<|\"|>text<|\"|>},path:<|\"|>f<|\"|>}<tool_call|>")
.tools({ shape_tool })
.expect(message_with_tool_calls("shape", R"({"edits":[],"headers":{"content-type":"text"},"path":"f"})"))
.run();
// Edge cases
tst.test(
"<|channel>thought\n<channel|>Hello, world!\nWhat's up?<channel|>")