chat : refactor gemma4 to build loose schema rules and clean up (#31915)

* chat : build a schema-shaped PEG parser for Gemma 4 tool arguments

* chat : enforce only the key structure of Gemma 4 tool arguments

* chat : remove old gemma4 workarounds

* cont : add node.as<>() to simplify static casts

* cont : add depth_guard and use numerical rule names

* cont : clean up

* cont : extract gemma 4 parsers

* cont : clean up

* cont : clean up

* cont : remove dynamic cast
This commit is contained in:
Aldehir Rojas authored and GitHub committed 2026-11-12 09:44:42 +02:00
1 parent 4fffab623e
commit 93cf736a8d
8 files changed
+426 -517

No files matched your search

+42 -44
View File
@@ -973,6 +973,21 @@ void common_chat_peg_gemma4_mapper::from_ast(const common_peg_ast_arena & arena,
}
}
using gemma4 = common_chat_peg_gemma4_mapper;
// Collects the nearest descendants carrying tag, looking through the rules the parser
// wraps around the tagged nodes.
static void gemma4_collect(const common_peg_ast_arena & arena, const common_peg_ast_node & node, const std::string & tag, std::vector<common_peg_ast_id> & out) {
for (auto child_id : node.children) {
const auto & child = arena.get(child_id);
if (child.tag == tag) {
out.push_back(child_id);
} else {
gemma4_collect(arena, child, tag, out);
}
}
}
static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg_ast_id id) {
const auto & node = arena.get(id);
@@ -980,51 +995,45 @@ static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg
return "";
}
if (node.rule == "gemma4-number" || node.rule == "gemma4-bool" || node.rule == "gemma4-null") {
if (node.tag == gemma4::NUMBER || node.tag == gemma4::BOOL || node.tag == gemma4::NULL_) {
return std::string(node.text);
}
if (node.rule == "gemma4-string-content") {
if (node.tag == gemma4::STRING_CONTENT) {
return escape_json_string_inner(node.sanitized_text());
}
if (node.rule == "gemma4-string") {
if (node.tag == gemma4::STRING) {
std::string result = "\"";
if (!node.children.empty()) {
result += gemma4_to_json(arena, node.children[0]);
if (!node.is_partial) {
result += "\"";
}
for (auto child_id : node.children) {
result += gemma4_to_json(arena, child_id);
}
if (!node.is_partial) {
result += "\"";
}
return result;
}
if (node.rule == "gemma4-array") {
if (node.tag == gemma4::ARRAY) {
std::string result = "[";
bool add_comma = false;
for (auto child_id : node.children) {
if (add_comma) {
for (size_t i = 0; i < node.children.size(); i++) {
if (i > 0) {
result += ',';
}
add_comma = true;
result += gemma4_to_json(arena, child_id);
result += gemma4_to_json(arena, node.children[i]);
}
if (!node.is_partial) {
result += ']';
}
return result;
}
if (node.rule == "gemma4-dict-key-name") {
return std::string(node.text);
}
if (node.tag == gemma4::DICT_KEY) {
auto name_id = arena.find_by_tag(node, gemma4::DICT_KEY_NAME);
if (node.rule == "gemma4-dict-key") {
std::string result = "\"";
if (!node.children.empty()) {
result += escape_json_string_inner(gemma4_to_json(arena, node.children[0]));
if (name_id != COMMON_PEG_INVALID_AST_ID) {
result += escape_json_string_inner(std::string(arena.get(name_id).text));
}
if (!node.is_partial) {
result += "\":";
@@ -1032,40 +1041,29 @@ static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg
return result;
}
if (node.rule == "gemma4-dict-kv") {
std::string result;
for (auto child_id : node.children) {
result += gemma4_to_json(arena, child_id);
}
return result;
}
if (node.tag == gemma4::DICT) {
std::vector<common_peg_ast_id> members;
gemma4_collect(arena, node, gemma4::DICT_KV, members);
if (node.rule == "gemma4-dict") {
std::string result = "{";
bool add_comma = false;
for (auto child_id : node.children) {
if (add_comma) {
for (size_t i = 0; i < members.size(); i++) {
if (i > 0) {
result += ',';
}
add_comma = true;
result += gemma4_to_json(arena, child_id);
result += gemma4_to_json(arena, members[i]);
}
if (!node.is_partial) {
result += '}';
}
return result;
}
if (node.rule == "gemma4-value") {
if (!node.children.empty()) {
return gemma4_to_json(arena, node.children[0]);
}
return "";
// members, rules and other wrappers
std::string result;
for (auto child_id : node.children) {
result += gemma4_to_json(arena, child_id);
}
return "";
return result;
}
void common_chat_peg_gemma4_mapper::visit(const common_peg_ast_arena & arena, common_peg_ast_id id) {