From 93cf736a8d1d46aa922e27eb64aa712bcd5ebc30 Mon Sep 17 00:00:00 2001 From: Aldehir Rojas Date: Thu, 12 Nov 2026 01:44:42 -0600 Subject: [PATCH] chat : refactor gemma4 to build loose schema rules and clean up (#31915) * chat : build a schema-shaped PEG parser for Gemma 4 tool arguments * chat : enforce only the key structure of Gemma 4 tool arguments * chat : remove old gemma4 workarounds * cont : add node.as<>() to simplify static casts * cont : add depth_guard and use numerical rule names * cont : clean up * cont : extract gemma 4 parsers * cont : clean up * cont : clean up * cont : remove dynamic cast --- common/chat-peg-parser.cpp | 86 ++-- common/chat-peg-parser.h | 12 + common/chat.cpp | 12 - common/json-schema.h | 62 ++- common/parsers/gemma4.cpp | 428 +++++++++++------- common/parsers/parsers.h | 6 - .../google-gemma-4-31B-it-interleaved.jinja | 282 ------------ tests/test-chat.cpp | 55 +++ 8 files changed, 426 insertions(+), 517 deletions(-) delete mode 100644 models/templates/google-gemma-4-31B-it-interleaved.jinja diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 4440b2ebd..968cbfa98 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -973,6 +973,21 @@ void common_chat_peg_gemma4_mapper::from_ast(const common_peg_ast_arena & arena, } } +using gemma4 = common_chat_peg_gemma4_mapper; + +// Collects the nearest descendants carrying tag, looking through the rules the parser +// wraps around the tagged nodes. +static void gemma4_collect(const common_peg_ast_arena & arena, const common_peg_ast_node & node, const std::string & tag, std::vector & out) { + for (auto child_id : node.children) { + const auto & child = arena.get(child_id); + if (child.tag == tag) { + out.push_back(child_id); + } else { + gemma4_collect(arena, child, tag, out); + } + } +} + static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg_ast_id id) { const auto & node = arena.get(id); @@ -980,51 +995,45 @@ static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg return ""; } - if (node.rule == "gemma4-number" || node.rule == "gemma4-bool" || node.rule == "gemma4-null") { + if (node.tag == gemma4::NUMBER || node.tag == gemma4::BOOL || node.tag == gemma4::NULL_) { return std::string(node.text); } - if (node.rule == "gemma4-string-content") { + if (node.tag == gemma4::STRING_CONTENT) { return escape_json_string_inner(node.sanitized_text()); } - if (node.rule == "gemma4-string") { + if (node.tag == gemma4::STRING) { std::string result = "\""; - if (!node.children.empty()) { - result += gemma4_to_json(arena, node.children[0]); - if (!node.is_partial) { - result += "\""; - } + for (auto child_id : node.children) { + result += gemma4_to_json(arena, child_id); + } + if (!node.is_partial) { + result += "\""; } return result; } - if (node.rule == "gemma4-array") { + if (node.tag == gemma4::ARRAY) { std::string result = "["; - - bool add_comma = false; - for (auto child_id : node.children) { - if (add_comma) { + for (size_t i = 0; i < node.children.size(); i++) { + if (i > 0) { result += ','; } - add_comma = true; - result += gemma4_to_json(arena, child_id); + result += gemma4_to_json(arena, node.children[i]); } - if (!node.is_partial) { result += ']'; } return result; } - if (node.rule == "gemma4-dict-key-name") { - return std::string(node.text); - } + if (node.tag == gemma4::DICT_KEY) { + auto name_id = arena.find_by_tag(node, gemma4::DICT_KEY_NAME); - if (node.rule == "gemma4-dict-key") { std::string result = "\""; - if (!node.children.empty()) { - result += escape_json_string_inner(gemma4_to_json(arena, node.children[0])); + if (name_id != COMMON_PEG_INVALID_AST_ID) { + result += escape_json_string_inner(std::string(arena.get(name_id).text)); } if (!node.is_partial) { result += "\":"; @@ -1032,40 +1041,29 @@ static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg return result; } - if (node.rule == "gemma4-dict-kv") { - std::string result; - for (auto child_id : node.children) { - result += gemma4_to_json(arena, child_id); - } - return result; - } + if (node.tag == gemma4::DICT) { + std::vector members; + gemma4_collect(arena, node, gemma4::DICT_KV, members); - if (node.rule == "gemma4-dict") { std::string result = "{"; - - bool add_comma = false; - for (auto child_id : node.children) { - if (add_comma) { + for (size_t i = 0; i < members.size(); i++) { + if (i > 0) { result += ','; } - add_comma = true; - result += gemma4_to_json(arena, child_id); + result += gemma4_to_json(arena, members[i]); } - if (!node.is_partial) { result += '}'; } return result; } - if (node.rule == "gemma4-value") { - if (!node.children.empty()) { - return gemma4_to_json(arena, node.children[0]); - } - return ""; + // members, rules and other wrappers + std::string result; + for (auto child_id : node.children) { + result += gemma4_to_json(arena, child_id); } - - return ""; + return result; } void common_chat_peg_gemma4_mapper::visit(const common_peg_ast_arena & arena, common_peg_ast_id id) { diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index 114fa049f..b827b60b0 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -34,6 +34,18 @@ class common_chat_peg_mapper { class common_chat_peg_gemma4_mapper : public common_chat_peg_mapper { public: + // Tags of the Gemma 4 argument syntax. The parser may wrap them in any rules. + static constexpr const char * STRING = "gemma4-string"; + static constexpr const char * STRING_CONTENT = "gemma4-string-content"; + static constexpr const char * NUMBER = "gemma4-number"; + static constexpr const char * BOOL = "gemma4-bool"; + static constexpr const char * NULL_ = "gemma4-null"; + static constexpr const char * ARRAY = "gemma4-array"; + static constexpr const char * DICT = "gemma4-dict"; + static constexpr const char * DICT_KV = "gemma4-dict-kv"; + static constexpr const char * DICT_KEY = "gemma4-dict-key"; + static constexpr const char * DICT_KEY_NAME = "gemma4-dict-key-name"; + common_chat_peg_gemma4_mapper(common_chat_msg & msg) : common_chat_peg_mapper(msg) {} virtual void from_ast(const common_peg_ast_arena & arena, const common_peg_parse_result & result); private: diff --git a/common/chat.cpp b/common/chat.cpp index 0e27f8197..da29cd671 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1277,13 +1277,6 @@ static common_chat_params common_chat_params_init_lfm2_5(const common_chat_templ return common_chat_params_init_lfm2(tmpl, inputs, /* tool_list_tokens = */ false); } -// Older gemma4 templates need their tool responses rewritten before rendering -static common_chat_params common_chat_params_init_gemma4_legacy(const common_chat_template & tmpl, const autoparser::generation_params & inputs) { - auto adjusted = inputs; - workaround::convert_tool_responses_gemma4(adjusted.messages); - return common_chat_params_init_gemma4(tmpl, adjusted); -} - // Pick the dedicated handler for a template from its source, or null for the autoparser. // Order matters: the first match wins, and later checks assume the earlier ones did not match. static common_chat_params_init_fn common_chat_template_detect_params_init(const std::string & src) { @@ -1418,11 +1411,6 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const // Gemma4 format detection if (src.find("'<|tool_call>call:'") != std::string::npos) { LOG_DBG("Using specialized template: Gemma4\n"); - if (src.find("{#- OpenAI Chat Completions:") == std::string::npos) { - LOG_WRN("%s: detected an outdated gemma4 chat template, applying compatibility workarounds. " - "Consider updating to the official template.\n", __func__); - return common_chat_params_init_gemma4_legacy; - } return common_chat_params_init_gemma4; } diff --git a/common/json-schema.h b/common/json-schema.h index 084208c96..b203e7777 100644 --- a/common/json-schema.h +++ b/common/json-schema.h @@ -85,92 +85,124 @@ struct common_chat_schema { static const char * kind_name(node_kind kind); static const char * type_name(value_type type); + + template + const T & as() const { + GGML_ASSERT(kind() == T::KIND); + return static_cast(*this); + } }; using common_chat_schema_ptr = std::unique_ptr; struct common_chat_schema_any : common_chat_schema { - node_kind kind() const override { return KIND_ANY; } + static constexpr node_kind KIND = KIND_ANY; + + node_kind kind() const override { return KIND; } }; // {"$ref": "#/..."}, only references into the same document are supported struct common_chat_schema_ref : common_chat_schema { + static constexpr node_kind KIND = KIND_REF; + std::string ref; const common_chat_schema * target = nullptr; // owned by common_chat_schema_document::refs explicit common_chat_schema_ref(std::string ref) : ref(std::move(ref)) {} - node_kind kind() const override { return KIND_REF; } + node_kind kind() const override { return KIND; } }; // oneOf / anyOf, or a "type" array expanded to one alternative per type struct common_chat_schema_any_of : common_chat_schema { + static constexpr node_kind KIND = KIND_ANY_OF; + std::vector children; - node_kind kind() const override { return KIND_ANY_OF; } + node_kind kind() const override { return KIND; } }; struct common_chat_schema_all_of : common_chat_schema { + static constexpr node_kind KIND = KIND_ALL_OF; + std::vector children; - node_kind kind() const override { return KIND_ALL_OF; } + node_kind kind() const override { return KIND; } }; struct common_chat_schema_const : common_chat_schema { + static constexpr node_kind KIND = KIND_CONST; + common_json value; explicit common_chat_schema_const(common_json value) : value(std::move(value)) {} - node_kind kind() const override { return KIND_CONST; } + node_kind kind() const override { return KIND; } }; struct common_chat_schema_enum : common_chat_schema { + static constexpr node_kind KIND = KIND_ENUM; + std::vector values; - node_kind kind() const override { return KIND_ENUM; } + node_kind kind() const override { return KIND; } }; struct common_chat_schema_null : common_chat_schema { - node_kind kind() const override { return KIND_NULL; } + static constexpr node_kind KIND = KIND_NULL; + + node_kind kind() const override { return KIND; } }; struct common_chat_schema_boolean : common_chat_schema { - node_kind kind() const override { return KIND_BOOLEAN; } + static constexpr node_kind KIND = KIND_BOOLEAN; + + node_kind kind() const override { return KIND; } }; struct common_chat_schema_number : common_chat_schema { - node_kind kind() const override { return KIND_NUMBER; } + static constexpr node_kind KIND = KIND_NUMBER; + + node_kind kind() const override { return KIND; } }; // bounds are inclusive, exclusiveMinimum / exclusiveMaximum are folded in struct common_chat_schema_integer : common_chat_schema { + static constexpr node_kind KIND = KIND_INTEGER; + int64_t minimum = INT64_MIN; // INT64_MIN for unbounded int64_t maximum = INT64_MAX; // INT64_MAX for unbounded - node_kind kind() const override { return KIND_INTEGER; } + node_kind kind() const override { return KIND; } }; struct common_chat_schema_string : common_chat_schema { + static constexpr node_kind KIND = KIND_STRING; + std::string pattern; // empty when absent string_format format = FORMAT_NONE; int min_length = 0; int max_length = -1; // -1 for unbounded - node_kind kind() const override { return KIND_STRING; } + node_kind kind() const override { return KIND; } }; struct common_chat_schema_array : common_chat_schema { + static constexpr node_kind KIND = KIND_ARRAY; + common_chat_schema_ptr items; // a common_chat_schema_any when "items" is absent int min_items = 0; int max_items = -1; // -1 for unbounded - node_kind kind() const override { return KIND_ARRAY; } + node_kind kind() const override { return KIND; } }; struct common_chat_schema_tuple : common_chat_schema { + static constexpr node_kind KIND = KIND_TUPLE; + std::vector items; - node_kind kind() const override { return KIND_TUPLE; } + node_kind kind() const override { return KIND; } }; struct common_chat_schema_property { @@ -180,10 +212,12 @@ struct common_chat_schema_property { }; struct common_chat_schema_object : common_chat_schema { + static constexpr node_kind KIND = KIND_OBJECT; + std::vector properties; // in schema order common_chat_schema_ptr additional_properties; // null when not allowed - node_kind kind() const override { return KIND_OBJECT; } + node_kind kind() const override { return KIND; } }; struct common_chat_schema_document { diff --git a/common/parsers/gemma4.cpp b/common/parsers/gemma4.cpp index 1a63fa927..5902e290b 100644 --- a/common/parsers/gemma4.cpp +++ b/common/parsers/gemma4.cpp @@ -1,152 +1,284 @@ #include "parsers.h" -namespace workaround { +#include "log.h" -// Gemma4 uses a custom tool_responses field instead of role:tool messages. -// -// This will transform a sequence of messages: -// assistant(tool_call+) -> tool+ -> assistant(content) -// -// Into a single assistant message containing a tool_responses field: -// assistant(content + tool_call + tool_responses) -// -// This is necessary for the Gemma4 chat template to properly format the prompt. -// See https://ai.google.dev/gemma/docs/core/prompt-formatting-gemma4 -struct gemma4_model_turn_builder { - json & messages; - size_t pos; - json tool_calls = json::array(); - json tool_responses = json::array(); - json content; - json reasoning_content; +#include +#include +#include +#include - gemma4_model_turn_builder(json & msgs, size_t pos) : messages(msgs), pos(pos) {} +namespace { - void collect() { - // Collect the first assistant message - auto & msg = messages[pos]; - if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) { - // According to the prompt formatting guide, we need to preserve reasoning_content - // between function calls. The current chat templates do not support this, but we will do it anyway. - reasoning_content = msg.at("reasoning_content"); +using gemma4 = common_chat_peg_gemma4_mapper; + +struct depth_guard { + int & depth; + depth_guard(int & depth) : depth(depth) { + if (++depth > COMMON_RECURSIVE_LIMIT) { + throw std::invalid_argument("JSON schema is nested more than " + std::to_string(COMMON_RECURSIVE_LIMIT) + " levels deep"); } - for (auto & tc : msg.at("tool_calls")) { - tool_calls.push_back(tc); - } - pos++; + } + ~depth_guard() { depth--; } +}; - // Collect tool call results - while (pos < messages.size() && messages[pos].value("role", "") == "tool") { - collect_result(messages[pos]); - pos++; - } +// Gemma 4 value parsers +void gemma4_value_rules(common_chat_peg_builder & p) { + // The tags are what gemma4_to_json() in the mapper reads, the rules may wrap them freely. + p.rule("gemma4-string", p.tag(gemma4::STRING, p.literal("<|\"|>") + p.tag(gemma4::STRING_CONTENT, p.until("<|\"|>")) + p.literal("<|\"|>"))); - // Check if the next assistant message is the final message - if (pos < messages.size() && messages[pos].value("role", "") == "assistant") { - auto & next = messages[pos]; - if (!has_tool_calls(next) && has_content(next)) { - content = next.at("content"); - pos++; - } + p.rule("gemma4-bool", p.tag(gemma4::BOOL, p.json_bool())); + + p.rule("gemma4-null", p.tag(gemma4::NULL_, p.json_null())); + + p.rule("gemma4-number", p.tag(gemma4::NUMBER, p.json_number())); + + // Keys are bare in the template, but Gemma 4 occasionally wraps them in quotes + p.rule("gemma4-dict-key", p.tag(gemma4::DICT_KEY, p.choice({ + p.literal("<|\"|>") + p.tag(gemma4::DICT_KEY_NAME, p.until("<|\"|>")) + p.literal("<|\"|>"), + p.literal("\"") + p.tag(gemma4::DICT_KEY_NAME, p.chars("[^\"]", 1, -1)) + p.literal("\""), + p.literal("'") + p.tag(gemma4::DICT_KEY_NAME, p.chars("[^']", 1, -1)) + p.literal("'"), + p.tag(gemma4::DICT_KEY_NAME, p.chars("[^:}]", 1, -1)), + }) + p.literal(":"))); + + p.rule("gemma4-dict-kv", p.tag(gemma4::DICT_KV, p.ref("gemma4-dict-key") + p.space() + p.ref("gemma4-value"))); + + p.rule("gemma4-dict", [&]() { + auto ws = p.space(); + auto member = p.ref("gemma4-dict-kv"); + auto members = p.sequence({member, p.zero_or_more(p.sequence({p.literal(","), ws, member}))}); + return p.tag(gemma4::DICT, p.sequence({ + p.literal("{"), ws, + p.choice({p.literal("}"), p.sequence({members, ws, p.literal("}")})}) + })); + }); + + p.rule("gemma4-array", [&]() { + auto ws = p.space(); + auto value = p.ref("gemma4-value"); + auto elements = p.sequence({value, p.zero_or_more(p.sequence({p.literal(","), ws, value}))}); + return p.tag(gemma4::ARRAY, p.sequence({ + p.literal("["), ws, + p.choice({p.literal("]"), p.sequence({elements, ws, p.literal("]")})}) + })); + }); + + p.rule("gemma4-value", [&]() { + return p.choice({ + p.ref("gemma4-string"), p.ref("gemma4-dict"), p.ref("gemma4-array"), + p.ref("gemma4-number"), p.ref("gemma4-bool"), p.ref("gemma4-null") + }); + }); +} + +// Gemma 4 uses different delimiters and cannot leverage the json-schema-to-grammar implementation. +// This implements a subset to enforce properties. +class gemma4_schema_parser { + common_chat_peg_builder & p_; + std::string root_; + int depth_ = 0; + + std::map refs_; + + public: + gemma4_schema_parser(common_chat_peg_builder & p, std::string root) : p_(p), root_(std::move(root)) {} + + common_peg_parser visit(const common_chat_schema & node) { return visit(node, root_); } + + private: + common_peg_parser visit(const common_chat_schema & node, const std::string & name) { + depth_guard guard(depth_); + switch (node.kind()) { + case common_chat_schema::KIND_REF: return visit_ref(node.as()); + case common_chat_schema::KIND_ANY_OF: return visit_any_of(node.as(), name); + case common_chat_schema::KIND_ALL_OF: return visit_all_of(node.as(), name); + case common_chat_schema::KIND_ARRAY: return visit_array(node.as(), name); + case common_chat_schema::KIND_TUPLE: return visit_tuple(node.as(), name); + case common_chat_schema::KIND_OBJECT: return visit_object(node.as(), name); + default: return p_.ref("gemma4-value"); } } - void collect_result(const json & curr) { - json response; - if (curr.contains("content")) { - const auto & content = curr.at("content"); - if (content.is_string()) { - // Try to parse the content as JSON; fall back to raw string - try { - response = json::parse(content.get()); - } catch (...) { - response = content; + // Whether a schema contributes a shape of its own, as opposed to accepting any value. + static bool shaped(const common_chat_schema & node) { + switch (node.kind()) { + case common_chat_schema::KIND_REF: + return node.as().target != nullptr; + case common_chat_schema::KIND_ANY_OF: + for (const auto & child : node.as().children) { + if (shaped(*child)) { + return true; + } } + return false; + case common_chat_schema::KIND_ALL_OF: { + const auto & children = node.as().children; + return children.size() == 1 && shaped(*children[0]); + } + case common_chat_schema::KIND_ARRAY: + case common_chat_schema::KIND_TUPLE: + case common_chat_schema::KIND_OBJECT: + return true; + default: + return false; + } + } + + // The template writes keys bare. Models keep that for identifiers and quote anything else, + // with single or double quotes or the string delimiter, so those are accepted as well. + common_peg_parser key(const std::string & key) { + auto name = [&]() { return p_.tag(gemma4::DICT_KEY_NAME, p_.literal(key)); }; + + bool identifier = !key.empty() && !std::isdigit(static_cast(key[0])); + for (char c : key) { + if (!std::isalnum(static_cast(c)) && c != '_') { + identifier = false; + } + } + if (identifier) { + return p_.tag(gemma4::DICT_KEY, name() + p_.literal(":")); + } + + auto alts = p_.choice(); + if (!key.empty() && key.find_first_of(":}") == std::string::npos && std::string("'\"<").find(key[0]) == std::string::npos) { + // Check if key is unambiguous + alts |= name(); + } + // Gemma 4 may emit keys in its string delimiters or quotes + alts |= p_.literal("<|\"|>") + name() + p_.literal("<|\"|>"); + if (key.find('\'') == std::string::npos) { + alts |= p_.literal("'") + name() + p_.literal("'"); + } + if (key.find('"') == std::string::npos) { + alts |= p_.literal("\"") + name() + p_.literal("\""); + } + return p_.tag(gemma4::DICT_KEY, alts + p_.literal(":")); + } + + common_peg_parser visit_ref(const common_chat_schema_ref & ref) { + if (!ref.target) { + return p_.ref("gemma4-value"); + } + auto it = refs_.find(ref.target); + if (it != refs_.end()) { + return p_.ref(it->second); + } + // A target reached from several places gets one rule, named by the order it was first seen. + auto rule = root_ + "-ref-" + std::to_string(refs_.size()); + refs_[ref.target] = rule; + return p_.rule(rule, [&]() { return visit(*ref.target, rule + "-value"); }); + } + + // Shaped alternatives are tried first, then anything else collapses into one generic value. + common_peg_parser visit_any_of(const common_chat_schema_any_of & node, const std::string & name) { + if (!shaped(node)) { + return p_.ref("gemma4-value"); + } + if (node.children.size() == 1) { + return visit(*node.children[0], name); + } + auto alts = p_.choice(); + bool loose = false; + for (size_t i = 0; i < node.children.size(); i++) { + if (shaped(*node.children[i])) { + alts |= visit(*node.children[i], name + "-" + std::to_string(i)); } else { - response = content; + loose = true; } } - - std::string name; - - // Match name with corresponding tool call - size_t idx = tool_responses.size(); - if (idx < tool_calls.size()) { - auto & tc = tool_calls[idx]; - if (tc.contains("function")) { - name = tc.at("function").value("name", ""); - } + if (loose) { + alts |= p_.ref("gemma4-value"); } - - // Fallback to the tool call id - if (name.empty()) { - name = curr.value("tool_call_id", ""); - } - - tool_responses.push_back({{"name", name}, {"response", response}}); + return p_.rule(name, alts); } - json build() { - collect(); + common_peg_parser visit_all_of(const common_chat_schema_all_of & node, const std::string & name) { + if (node.children.size() == 1) { + return visit(*node.children[0], name); + } + return p_.ref("gemma4-value"); + } - json msg = { - {"role", "assistant"}, - {"tool_calls", tool_calls}, + common_peg_parser visit_array(const common_chat_schema_array & node, const std::string & name) { + auto ws = p_.space(); + auto item = node.items && shaped(*node.items) ? visit(*node.items, name + "-item") : p_.ref("gemma4-value"); + auto elements = item + p_.zero_or_more(p_.literal(",") + ws + item); + return p_.rule(name, p_.tag(gemma4::ARRAY, + p_.literal("[") + ws + (p_.literal("]") | (elements + ws + p_.literal("]"))))); + } + + common_peg_parser visit_tuple(const common_chat_schema_tuple & node, const std::string & name) { + auto ws = p_.space(); + auto items = p_.sequence(); + for (size_t i = 0; i < node.items.size(); i++) { + if (i > 0) { + items = items + p_.literal(",") + ws; + } + items = items + visit(*node.items[i], name + "-" + std::to_string(i)); + } + return p_.rule(name, p_.tag(gemma4::ARRAY, p_.literal("[") + ws + items + ws + p_.literal("]"))); + } + + common_peg_parser visit_object(const common_chat_schema_object & node, const std::string & name) { + auto ws = p_.space(); + + std::vector props; + for (const auto & prop : node.properties) { + props.push_back(&prop); + } + + // The Gemma 4 jinja template sorts properties, so our rules need to match. + auto lower = [](const std::string & s) { + std::string out = s; + for (auto & c : out) { + c = static_cast(std::tolower(static_cast(c))); + } + return out; }; - if (!tool_responses.empty()) { - msg["tool_responses"] = tool_responses; - } - if (!content.is_null()) { - msg["content"] = content; - } - if (!reasoning_content.is_null()) { - msg["reasoning_content"] = reasoning_content; - } - return msg; - } + std::stable_sort(props.begin(), props.end(), [&](const auto * a, const auto * b) { + return lower(a->name) < lower(b->name); + }); - static bool has_content(const json & msg) { - if (!msg.contains("content") || msg.at("content").is_null()) { - return false; + if (props.empty()) { + if (node.additional_properties) { + return p_.ref("gemma4-dict"); + } + return p_.rule(name, p_.tag(gemma4::DICT, p_.literal("{") + ws + p_.literal("}"))); } - const auto & content = msg.at("content"); - if (content.is_string() && !content.get().empty()) { - return true; - } - if (content.is_array() && !content.empty()) { - return true; - } - return false; - } - static bool has_tool_calls(const json & msg) { - return msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty(); + std::optional rest; + std::optional first; + if (node.additional_properties) { + auto kv = p_.ref("gemma4-dict-kv"); + rest = p_.zero_or_more(p_.literal(",") + ws + kv); + first = p_.optional(kv + *rest); + } + + auto then_rest = [&](const common_peg_parser & parser) { + return rest ? parser + *rest : parser; + }; + + for (size_t i = props.size(); i-- > 0;) { + const auto & prop = *props[i]; + auto kv = p_.tag(gemma4::DICT_KV, key(prop.name) + ws + visit(*prop.schema, name + "-" + std::to_string(i))); + + common_peg_parser rest_i = then_rest(p_.literal(",") + ws + kv); + common_peg_parser first_i = then_rest(kv); + if (!prop.required) { + rest_i = then_rest(p_.optional(p_.literal(",") + ws + kv)); + first_i = first ? (first_i | *first) : p_.optional(first_i); + } + + if (i > 0) { + rest = p_.rule(name + "-rest-" + std::to_string(i), rest_i); + } + first = first_i; + } + + return p_.rule(name, p_.tag(gemma4::DICT, p_.literal("{") + ws + *first + ws + p_.literal("}"))); } }; -void convert_tool_responses_gemma4(json & messages) { - json result = json::array(); - size_t i = 0; - - while (i < messages.size()) { - auto & msg = messages[i]; - - if (msg.value("role", "") != "assistant" || !msg.contains("tool_calls") || - !msg.at("tool_calls").is_array() || msg.at("tool_calls").empty()) { - result.push_back(msg); - i++; - continue; - } - - gemma4_model_turn_builder builder(messages, i); - result.push_back(builder.build()); - i = builder.pos; - } - - messages = result; -} - -} +} // namespace common_chat_params common_chat_params_init_gemma4(const common_chat_template & tmpl, const autoparser::generation_params & inputs) { @@ -218,51 +350,29 @@ common_chat_params common_chat_params_init_gemma4(const common_chat_template & } if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - // Gemma4 tool calling syntax - // Rules should match traversal logic in gemma4_to_json() - p.rule("gemma4-string-content", p.until("<|\"|>")); - p.rule("gemma4-string", p.literal("<|\"|>") + p.ref("gemma4-string-content") + p.literal("<|\"|>")); - p.rule("gemma4-bool", p.json_bool()); - p.rule("gemma4-null", p.json_null()); - p.rule("gemma4-number", p.json_number()); - p.rule("gemma4-dict-key", p.rule("gemma4-dict-key-name", p.chars("[^:}]", 1, -1)) + p.literal(":")); - p.rule("gemma4-dict-kv", p.ref("gemma4-dict-key") + p.space() + p.ref("gemma4-value")); - p.rule("gemma4-dict", [&]() { - auto ws = p.space(); - auto member = p.ref("gemma4-dict-kv"); - auto members = p.sequence({member, p.zero_or_more(p.sequence({p.literal(","), ws, member}))}); - return p.sequence({ - p.literal("{"), ws, - p.choice({p.literal("}"), p.sequence({members, ws, p.literal("}")})}) - }); - }); - p.rule("gemma4-array", [&]() { - auto ws = p.space(); - auto value = p.ref("gemma4-value"); - auto elements = p.sequence({value, p.zero_or_more(p.sequence({p.literal(","), ws, value}))}); - return p.sequence({ - p.literal("["), ws, - p.choice({p.literal("]"), p.sequence({elements, ws, p.literal("]")})}) - }); - }); - p.rule("gemma4-value", [&]() { - return p.choice({ - p.ref("gemma4-string"), p.ref("gemma4-dict"), p.ref("gemma4-array"), - p.ref("gemma4-number"), p.ref("gemma4-bool"), p.ref("gemma4-null") - }); - }); + gemma4_value_rules(p); auto tool_choice = p.choice(); foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) { const auto & function = tool.at("function"); std::string name = function.at("name"); - // TODO @aldehir : need to extend json-schema-to-grammar to produce more than JSON rules - // const auto & params = function.at("parameters"); + + // Arguments follow the shape of the parameter schema when it is an object, + // see gemma4_schema_parser, and are a generic dict otherwise. + auto args = p.ref("gemma4-dict"); + try { + auto doc = common_chat_schema_from_json(common_chat_tool_parameters(function)); + if (doc.root->kind() == common_chat_schema::KIND_OBJECT) { + args = gemma4_schema_parser(p, "tool-" + std::to_string(tool_index) + "-args").visit(*doc.root); + } + } catch (const std::exception & e) { + LOG_WRN("%s: tool %s uses the generic argument parser: %s\n", __func__, name.c_str(), e.what()); + } tool_choice |= p.rule("tool-" + std::to_string(tool_index), p.tool(p.sequence({ p.tool_open(p.tool_name(p.literal(name)) + p.peek(p.literal("{"))), - p.tool_args(p.ref("gemma4-dict")), + p.tool_args(args), }))); }); diff --git a/common/parsers/parsers.h b/common/parsers/parsers.h index 532a44f01..b1b6d166a 100644 --- a/common/parsers/parsers.h +++ b/common/parsers/parsers.h @@ -41,12 +41,6 @@ std::string common_chat_template_generation_prompt_impl( bool is_lfm2_template(const std::string & src); -namespace workaround { - -void convert_tool_responses_gemma4(json & messages); - -} - common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl, const autoparser::generation_params & inputs); common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl, const autoparser::generation_params & inputs); diff --git a/models/templates/google-gemma-4-31B-it-interleaved.jinja b/models/templates/google-gemma-4-31B-it-interleaved.jinja deleted file mode 100644 index 85791c4fe..000000000 --- a/models/templates/google-gemma-4-31B-it-interleaved.jinja +++ /dev/null @@ -1,282 +0,0 @@ -{%- macro format_parameters(properties, required) -%} - {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%} - {%- set ns = namespace(found_first=false) -%} - {%- for key, value in properties | dictsort -%} - {%- set add_comma = false -%} - {%- if key not in standard_keys -%} - {%- if ns.found_first %},{% endif -%} - {%- set ns.found_first = true -%} - {{ key }}:{ - {%- if value['description'] -%} - description:<|"|>{{ value['description'] }}<|"|> - {%- set add_comma = true -%} - {%- endif -%} - {%- if value['nullable'] %} - {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} - nullable:true - {%- endif -%} - {%- if value['type'] | upper == 'STRING' -%} - {%- if value['enum'] -%} - {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} - enum:{{ format_argument(value['enum']) }} - {%- endif -%} - {%- elif value['type'] | upper == 'OBJECT' -%} - ,properties:{ - {%- if value['properties'] is defined and value['properties'] is mapping -%} - {{- format_parameters(value['properties'], value['required'] | default([])) -}} - {%- elif value is mapping -%} - {{- format_parameters(value, value['required'] | default([])) -}} - {%- endif -%} - } - {%- if value['required'] -%} - ,required:[ - {%- for item in value['required'] | default([]) -%} - <|"|>{{- item -}}<|"|> - {%- if not loop.last %},{% endif -%} - {%- endfor -%} - ] - {%- endif -%} - {%- elif value['type'] | upper == 'ARRAY' -%} - {%- if value['items'] is mapping and value['items'] -%} - ,items:{ - {%- set ns_items = namespace(found_first=false) -%} - {%- for item_key, item_value in value['items'] | dictsort -%} - {%- if item_value is not none -%} - {%- if ns_items.found_first %},{% endif -%} - {%- set ns_items.found_first = true -%} - {%- if item_key == 'properties' -%} - properties:{ - {%- if item_value is mapping -%} - {{- format_parameters(item_value, value['items']['required'] | default([])) -}} - {%- endif -%} - } - {%- elif item_key == 'required' -%} - required:[ - {%- for req_item in item_value -%} - <|"|>{{- req_item -}}<|"|> - {%- if not loop.last %},{% endif -%} - {%- endfor -%} - ] - {%- elif item_key == 'type' -%} - {%- if item_value is string -%} - type:{{ format_argument(item_value | upper) }} - {%- else -%} - type:{{ format_argument(item_value | map('upper') | list) }} - {%- endif -%} - {%- else -%} - {{ item_key }}:{{ format_argument(item_value) }} - {%- endif -%} - {%- endif -%} - {%- endfor -%} - } - {%- endif -%} - {%- endif -%} - {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} - type:<|"|>{{ value['type'] | upper }}<|"|>} - {%- endif -%} - {%- endfor -%} -{%- endmacro -%} -{%- macro format_function_declaration(tool_data) -%} - declaration:{{- tool_data['function']['name'] -}}{description:<|"|>{{- tool_data['function']['description'] -}}<|"|> - {%- set params = tool_data['function']['parameters'] -%} - {%- if params -%} - ,parameters:{ - {%- if params['properties'] -%} - properties:{ {{- format_parameters(params['properties'], params['required']) -}} }, - {%- endif -%} - {%- if params['required'] -%} - required:[ - {%- for item in params['required'] -%} - <|"|>{{- item -}}<|"|> - {{- ',' if not loop.last -}} - {%- endfor -%} - ], - {%- endif -%} - {%- if params['type'] -%} - type:<|"|>{{- params['type'] | upper -}}<|"|>} - {%- endif -%} - {%- endif -%} - {%- if 'response' in tool_data['function'] -%} - {%- set response_declaration = tool_data['function']['response'] -%} - ,response:{ - {%- if response_declaration['description'] -%} - description:<|"|>{{- response_declaration['description'] -}}<|"|>, - {%- endif -%} - {%- if response_declaration['type'] | upper == 'OBJECT' -%} - type:<|"|>{{- response_declaration['type'] | upper -}}<|"|>} - {%- endif -%} - {%- endif -%} - } -{%- endmacro -%} -{%- macro format_argument(argument, escape_keys=True) -%} - {%- if argument is string -%} - {{- '<|"|>' + argument + '<|"|>' -}} - {%- elif argument is boolean -%} - {{- 'true' if argument else 'false' -}} - {%- elif argument is mapping -%} - {{- '{' -}} - {%- set ns = namespace(found_first=false) -%} - {%- for key, value in argument | dictsort -%} - {%- if ns.found_first %},{% endif -%} - {%- set ns.found_first = true -%} - {%- if escape_keys -%} - {{- '<|"|>' + key + '<|"|>' -}} - {%- else -%} - {{- key -}} - {%- endif -%} - :{{- format_argument(value, escape_keys=escape_keys) -}} - {%- endfor -%} - {{- '}' -}} - {%- elif argument is sequence -%} - {{- '[' -}} - {%- for item in argument -%} - {{- format_argument(item, escape_keys=escape_keys) -}} - {%- if not loop.last %},{% endif -%} - {%- endfor -%} - {{- ']' -}} - {%- else -%} - {{- argument -}} - {%- endif -%} -{%- endmacro -%} -{%- macro strip_thinking(text) -%} - {%- set ns = namespace(result='') -%} - {%- for part in text.split('') -%} - {%- if '<|channel>' in part -%} - {%- set ns.result = ns.result + part.split('<|channel>')[0] -%} - {%- else -%} - {%- set ns.result = ns.result + part -%} - {%- endif -%} - {%- endfor -%} - {{- ns.result | trim -}} -{%- endmacro -%} - -{%- set ns = namespace(prev_message_type=None, last_user_message=-1) -%} -{%- set loop_messages = messages -%} -{{- bos_token -}} -{#- Handle System/Tool Definitions Block -#} -{%- if (enable_thinking is defined and enable_thinking) or tools or messages[0]['role'] in ['system', 'developer'] -%} - {{- '<|turn>system\n' -}} - - {#- Inject Thinking token at the very top of the FIRST system turn -#} - {%- if enable_thinking is defined and enable_thinking -%} - {{- '<|think|>\n' -}} - {%- set ns.prev_message_type = 'think' -%} - {%- endif -%} - - {%- if messages[0]['role'] in ['system', 'developer'] -%} - {{- messages[0]['content'] | trim -}} - {%- set loop_messages = messages[1:] -%} - {%- endif -%} - - {%- if tools -%} - {%- for tool in tools %} - {{- '<|tool>' -}} - {{- format_function_declaration(tool) | trim -}} - {{- '' -}} - {%- endfor %} - {%- set ns.prev_message_type = 'tool' -%} - {%- endif -%} - - {{- '\n' -}} -{%- endif %} - -{#- Find last user message -#} -{%- for message in loop_messages -%} - {%- if message['role'] == 'user' -%} - {%- set ns.last_user_message = loop.index0 -%} - {%- endif -%} -{%- endfor -%} - -{#- Loop through messages -#} -{%- for message in loop_messages -%} - {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%} - {%- if not (ns.prev_message_type == 'tool_response' and message['tool_calls']) -%} - {{- '<|turn>' + role + '\n' }} - {%- endif -%} - - {%- set ns.prev_message_type = None -%} - - {%- if message['tool_calls'] -%} - {#- Preserve reasoning between tool calls for model turns that come after the last user turn -#} - {%- if message['reasoning_content'] and loop.index0 > ns.last_user_message -%} - {{- '<|channel>thought\n' -}} - {{- message['reasoning_content'] -}} - {{- '' -}} - {%- endif -%} - {%- for tool_call in message['tool_calls'] -%} - {%- set function = tool_call['function'] -%} - {{- '<|tool_call>call:' + function['name'] + '{' -}} - {%- if function['arguments'] is mapping -%} - {%- set ns_args = namespace(found_first=false) -%} - {%- for key, value in function['arguments'] | dictsort -%} - {%- if ns_args.found_first %},{% endif -%} - {%- set ns_args.found_first = true -%} - {{- key -}}:{{- format_argument(value, escape_keys=False) -}} - {%- endfor -%} - {%- elif function['arguments'] is string -%} - {{- function['arguments'] -}} - {%- endif -%} - {{- '}' -}} - {%- endfor -%} - {%- set ns.prev_message_type = 'tool_call' -%} - {%- endif -%} - - {%- if message['tool_responses'] -%} - {#- Tool Response handling -#} - {%- for tool_response in message['tool_responses'] -%} - {{- '<|tool_response>' -}} - {%- if tool_response['response'] is mapping -%} - {{- 'response:' + tool_response['name'] | default('unknown') + '{' -}} - {%- for key, value in tool_response['response'] | dictsort -%} - {{- key -}}:{{- format_argument(value, escape_keys=False) -}} - {%- if not loop.last %},{% endif -%} - {%- endfor -%} - {{- '}' -}} - {%- else -%} - {{- 'response:' + tool_response['name'] | default('unknown') + '{value:' + format_argument(tool_response['response'], escape_keys=False) + '}' -}} - {%- endif -%} - {{- '' -}} - {%- endfor -%} - {%- set ns.prev_message_type = 'tool_response' -%} - {%- endif -%} - - {%- if message['content'] is string -%} - {%- if role == 'model' -%} - {{- strip_thinking(message['content']) -}} - {%- else -%} - {{- message['content'] | trim -}} - {%- endif -%} - {%- elif message['content'] is sequence -%} - {%- for item in message['content'] -%} - {%- if item['type'] == 'text' -%} - {%- if role == 'model' -%} - {{- strip_thinking(item['text']) -}} - {%- else -%} - {{- item['text'] | trim -}} - {%- endif -%} - {%- elif item['type'] == 'image' -%} - {{- '<|image|>' -}} - {%- set ns.prev_message_type = 'image' -%} - {%- elif item['type'] == 'audio' -%} - {{- '<|audio|>' -}} - {%- set ns.prev_message_type = 'audio' -%} - {%- elif item['type'] == 'video' -%} - {{- '<|video|>' -}} - {%- set ns.prev_message_type = 'video' -%} - {%- endif -%} - {%- endfor -%} - {%- endif -%} - - {%- if not (message['tool_responses'] and not message['content']) -%} - {{- '\n' -}} - {%- endif -%} -{%- endfor -%} - -{%- if add_generation_prompt -%} - {%- if ns.prev_message_type != 'tool_response' -%} - {{- '<|turn>model\n' -}} - {%- endif -%} - {%- if not enable_thinking | default(false) -%} - {{- '<|channel>thought\n' -}} - {%- endif -%} -{%- endif -%} diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index b565775bf..2c59d1222 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -3267,6 +3267,61 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect(message_with_tool_calls("amount", R"({"orig": 1.5e10})")) .run(); + // The argument grammar follows the shape of the parameter schema: keys in + // alphabetical order, optional keys skipped in place, nested objects and arrays + // shaped the same way, any other value accepted as written. + common_chat_tool shape_tool{ + /* .name = */ "shape", + /* .description = */ "Schema-shaped arguments", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "path": {"type": "string"}, + "edits": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oldText": {"type": "string"}, + "newText": {"type": "string"} + }, + "required": ["oldText", "newText"] + } + }, + "mode": {"type": "string", "enum": ["fast", "safe"]}, + "count": {"type": "integer"}, + "headers": { + "type": "object", + "properties": { + "content-type": {"type": "string"}, + "accept:lang": {"type": "string"} + } + }, + "options": {"type": "object"} + }, + "required": ["path", "edits"] + })", + }; + + tst.test( + "<|tool_call>call:shape{edits:[{newText:<|\"|>b<|\"|>,oldText:<|\"|>a<|\"|>}],path:<|\"|>f.c<|\"|>}") + .tools({ shape_tool }) + .expect(message_with_tool_calls("shape", R"({"edits":[{"newText":"b","oldText":"a"}],"path":"f.c"})")) + .run(); + + // Optional keys in place, quoted keys that are not identifiers + tst.test( + "<|tool_call>call:shape{count:3,edits:[],headers:{<|\"|>accept:lang<|\"|>:<|\"|>en<|\"|>,'content-type':<|\"|>text<|\"|>},mode:<|\"|>fast<|\"|>,options:{b:1,a:[true]},path:<|\"|>f<|\"|>}") + .tools({ shape_tool }) + .expect(message_with_tool_calls("shape", R"({"count":3,"edits":[],"headers":{"accept:lang":"en","content-type":"text"},"mode":"fast","options":{"b":1,"a":[true]},"path":"f"})")) + .run(); + + tst.test( + "<|tool_call>call:shape{edits:[],headers:{\"content-type\":<|\"|>text<|\"|>},path:<|\"|>f<|\"|>}") + .tools({ shape_tool }) + .expect(message_with_tool_calls("shape", R"({"edits":[],"headers":{"content-type":"text"},"path":"f"})")) + .run(); + // Edge cases tst.test( "<|channel>thought\nHello, world!\nWhat's up?")