chat : refactor gemma4 to build loose schema rules and clean up (#31915)

* chat : build a schema-shaped PEG parser for Gemma 4 tool arguments

* chat : enforce only the key structure of Gemma 4 tool arguments

* chat : remove old gemma4 workarounds

* cont : add node.as<>() to simplify static casts

* cont : add depth_guard and use numerical rule names

* cont : clean up

* cont : extract gemma 4 parsers

* cont : clean up

* cont : clean up

* cont : remove dynamic cast
This commit is contained in:
Aldehir Rojas authored and GitHub committed 2026-11-12 09:44:42 +02:00
1 parent 4fffab623e
commit 93cf736a8d
8 files changed
+426 -517

No files matched your search

+42 -44
View File
@@ -973,6 +973,21 @@ void common_chat_peg_gemma4_mapper::from_ast(const common_peg_ast_arena & arena,
}
}
using gemma4 = common_chat_peg_gemma4_mapper;
// Collects the nearest descendants carrying tag, looking through the rules the parser
// wraps around the tagged nodes.
static void gemma4_collect(const common_peg_ast_arena & arena, const common_peg_ast_node & node, const std::string & tag, std::vector<common_peg_ast_id> & out) {
for (auto child_id : node.children) {
const auto & child = arena.get(child_id);
if (child.tag == tag) {
out.push_back(child_id);
} else {
gemma4_collect(arena, child, tag, out);
}
}
}
static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg_ast_id id) {
const auto & node = arena.get(id);
@@ -980,51 +995,45 @@ static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg
return "";
}
if (node.rule == "gemma4-number" || node.rule == "gemma4-bool" || node.rule == "gemma4-null") {
if (node.tag == gemma4::NUMBER || node.tag == gemma4::BOOL || node.tag == gemma4::NULL_) {
return std::string(node.text);
}
if (node.rule == "gemma4-string-content") {
if (node.tag == gemma4::STRING_CONTENT) {
return escape_json_string_inner(node.sanitized_text());
}
if (node.rule == "gemma4-string") {
if (node.tag == gemma4::STRING) {
std::string result = "\"";
if (!node.children.empty()) {
result += gemma4_to_json(arena, node.children[0]);
if (!node.is_partial) {
result += "\"";
}
for (auto child_id : node.children) {
result += gemma4_to_json(arena, child_id);
}
if (!node.is_partial) {
result += "\"";
}
return result;
}
if (node.rule == "gemma4-array") {
if (node.tag == gemma4::ARRAY) {
std::string result = "[";
bool add_comma = false;
for (auto child_id : node.children) {
if (add_comma) {
for (size_t i = 0; i < node.children.size(); i++) {
if (i > 0) {
result += ',';
}
add_comma = true;
result += gemma4_to_json(arena, child_id);
result += gemma4_to_json(arena, node.children[i]);
}
if (!node.is_partial) {
result += ']';
}
return result;
}
if (node.rule == "gemma4-dict-key-name") {
return std::string(node.text);
}
if (node.tag == gemma4::DICT_KEY) {
auto name_id = arena.find_by_tag(node, gemma4::DICT_KEY_NAME);
if (node.rule == "gemma4-dict-key") {
std::string result = "\"";
if (!node.children.empty()) {
result += escape_json_string_inner(gemma4_to_json(arena, node.children[0]));
if (name_id != COMMON_PEG_INVALID_AST_ID) {
result += escape_json_string_inner(std::string(arena.get(name_id).text));
}
if (!node.is_partial) {
result += "\":";
@@ -1032,40 +1041,29 @@ static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg
return result;
}
if (node.rule == "gemma4-dict-kv") {
std::string result;
for (auto child_id : node.children) {
result += gemma4_to_json(arena, child_id);
}
return result;
}
if (node.tag == gemma4::DICT) {
std::vector<common_peg_ast_id> members;
gemma4_collect(arena, node, gemma4::DICT_KV, members);
if (node.rule == "gemma4-dict") {
std::string result = "{";
bool add_comma = false;
for (auto child_id : node.children) {
if (add_comma) {
for (size_t i = 0; i < members.size(); i++) {
if (i > 0) {
result += ',';
}
add_comma = true;
result += gemma4_to_json(arena, child_id);
result += gemma4_to_json(arena, members[i]);
}
if (!node.is_partial) {
result += '}';
}
return result;
}
if (node.rule == "gemma4-value") {
if (!node.children.empty()) {
return gemma4_to_json(arena, node.children[0]);
}
return "";
// members, rules and other wrappers
std::string result;
for (auto child_id : node.children) {
result += gemma4_to_json(arena, child_id);
}
return "";
return result;
}
void common_chat_peg_gemma4_mapper::visit(const common_peg_ast_arena & arena, common_peg_ast_id id) {
+12
View File
@@ -34,6 +34,18 @@ class common_chat_peg_mapper {
class common_chat_peg_gemma4_mapper : public common_chat_peg_mapper {
public:
// Tags of the Gemma 4 argument syntax. The parser may wrap them in any rules.
static constexpr const char * STRING = "gemma4-string";
static constexpr const char * STRING_CONTENT = "gemma4-string-content";
static constexpr const char * NUMBER = "gemma4-number";
static constexpr const char * BOOL = "gemma4-bool";
static constexpr const char * NULL_ = "gemma4-null";
static constexpr const char * ARRAY = "gemma4-array";
static constexpr const char * DICT = "gemma4-dict";
static constexpr const char * DICT_KV = "gemma4-dict-kv";
static constexpr const char * DICT_KEY = "gemma4-dict-key";
static constexpr const char * DICT_KEY_NAME = "gemma4-dict-key-name";
common_chat_peg_gemma4_mapper(common_chat_msg & msg) : common_chat_peg_mapper(msg) {}
virtual void from_ast(const common_peg_ast_arena & arena, const common_peg_parse_result & result);
private:
-12
View File
@@ -1277,13 +1277,6 @@ static common_chat_params common_chat_params_init_lfm2_5(const common_chat_templ
return common_chat_params_init_lfm2(tmpl, inputs, /* tool_list_tokens = */ false);
}
// Older gemma4 templates need their tool responses rewritten before rendering
static common_chat_params common_chat_params_init_gemma4_legacy(const common_chat_template & tmpl, const autoparser::generation_params & inputs) {
auto adjusted = inputs;
workaround::convert_tool_responses_gemma4(adjusted.messages);
return common_chat_params_init_gemma4(tmpl, adjusted);
}
// Pick the dedicated handler for a template from its source, or null for the autoparser.
// Order matters: the first match wins, and later checks assume the earlier ones did not match.
static common_chat_params_init_fn common_chat_template_detect_params_init(const std::string & src) {
@@ -1418,11 +1411,6 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
// Gemma4 format detection
if (src.find("'<|tool_call>call:'") != std::string::npos) {
LOG_DBG("Using specialized template: Gemma4\n");
if (src.find("{#- OpenAI Chat Completions:") == std::string::npos) {
LOG_WRN("%s: detected an outdated gemma4 chat template, applying compatibility workarounds. "
"Consider updating to the official template.\n", __func__);
return common_chat_params_init_gemma4_legacy;
}
return common_chat_params_init_gemma4;
}
+48 -14
View File
@@ -85,92 +85,124 @@ struct common_chat_schema {
static const char * kind_name(node_kind kind);
static const char * type_name(value_type type);
template <typename T>
const T & as() const {
GGML_ASSERT(kind() == T::KIND);
return static_cast<const T &>(*this);
}
};
using common_chat_schema_ptr = std::unique_ptr<common_chat_schema>;
struct common_chat_schema_any : common_chat_schema {
node_kind kind() const override { return KIND_ANY; }
static constexpr node_kind KIND = KIND_ANY;
node_kind kind() const override { return KIND; }
};
// {"$ref": "#/..."}, only references into the same document are supported
struct common_chat_schema_ref : common_chat_schema {
static constexpr node_kind KIND = KIND_REF;
std::string ref;
const common_chat_schema * target = nullptr; // owned by common_chat_schema_document::refs
explicit common_chat_schema_ref(std::string ref) : ref(std::move(ref)) {}
node_kind kind() const override { return KIND_REF; }
node_kind kind() const override { return KIND; }
};
// oneOf / anyOf, or a "type" array expanded to one alternative per type
struct common_chat_schema_any_of : common_chat_schema {
static constexpr node_kind KIND = KIND_ANY_OF;
std::vector<common_chat_schema_ptr> children;
node_kind kind() const override { return KIND_ANY_OF; }
node_kind kind() const override { return KIND; }
};
struct common_chat_schema_all_of : common_chat_schema {
static constexpr node_kind KIND = KIND_ALL_OF;
std::vector<common_chat_schema_ptr> children;
node_kind kind() const override { return KIND_ALL_OF; }
node_kind kind() const override { return KIND; }
};
struct common_chat_schema_const : common_chat_schema {
static constexpr node_kind KIND = KIND_CONST;
common_json value;
explicit common_chat_schema_const(common_json value) : value(std::move(value)) {}
node_kind kind() const override { return KIND_CONST; }
node_kind kind() const override { return KIND; }
};
struct common_chat_schema_enum : common_chat_schema {
static constexpr node_kind KIND = KIND_ENUM;
std::vector<common_json> values;
node_kind kind() const override { return KIND_ENUM; }
node_kind kind() const override { return KIND; }
};
struct common_chat_schema_null : common_chat_schema {
node_kind kind() const override { return KIND_NULL; }
static constexpr node_kind KIND = KIND_NULL;
node_kind kind() const override { return KIND; }
};
struct common_chat_schema_boolean : common_chat_schema {
node_kind kind() const override { return KIND_BOOLEAN; }
static constexpr node_kind KIND = KIND_BOOLEAN;
node_kind kind() const override { return KIND; }
};
struct common_chat_schema_number : common_chat_schema {
node_kind kind() const override { return KIND_NUMBER; }
static constexpr node_kind KIND = KIND_NUMBER;
node_kind kind() const override { return KIND; }
};
// bounds are inclusive, exclusiveMinimum / exclusiveMaximum are folded in
struct common_chat_schema_integer : common_chat_schema {
static constexpr node_kind KIND = KIND_INTEGER;
int64_t minimum = INT64_MIN; // INT64_MIN for unbounded
int64_t maximum = INT64_MAX; // INT64_MAX for unbounded
node_kind kind() const override { return KIND_INTEGER; }
node_kind kind() const override { return KIND; }
};
struct common_chat_schema_string : common_chat_schema {
static constexpr node_kind KIND = KIND_STRING;
std::string pattern; // empty when absent
string_format format = FORMAT_NONE;
int min_length = 0;
int max_length = -1; // -1 for unbounded
node_kind kind() const override { return KIND_STRING; }
node_kind kind() const override { return KIND; }
};
struct common_chat_schema_array : common_chat_schema {
static constexpr node_kind KIND = KIND_ARRAY;
common_chat_schema_ptr items; // a common_chat_schema_any when "items" is absent
int min_items = 0;
int max_items = -1; // -1 for unbounded
node_kind kind() const override { return KIND_ARRAY; }
node_kind kind() const override { return KIND; }
};
struct common_chat_schema_tuple : common_chat_schema {
static constexpr node_kind KIND = KIND_TUPLE;
std::vector<common_chat_schema_ptr> items;
node_kind kind() const override { return KIND_TUPLE; }
node_kind kind() const override { return KIND; }
};
struct common_chat_schema_property {
@@ -180,10 +212,12 @@ struct common_chat_schema_property {
};
struct common_chat_schema_object : common_chat_schema {
static constexpr node_kind KIND = KIND_OBJECT;
std::vector<common_chat_schema_property> properties; // in schema order
common_chat_schema_ptr additional_properties; // null when not allowed
node_kind kind() const override { return KIND_OBJECT; }
node_kind kind() const override { return KIND; }
};
struct common_chat_schema_document {
+269 -159
View File
@@ -1,152 +1,284 @@
#include "parsers.h"
namespace workaround {
#include "log.h"
// Gemma4 uses a custom tool_responses field instead of role:tool messages.
//
// This will transform a sequence of messages:
// assistant(tool_call+) -> tool+ -> assistant(content)
//
// Into a single assistant message containing a tool_responses field:
// assistant(content + tool_call + tool_responses)
//
// This is necessary for the Gemma4 chat template to properly format the prompt.
// See https://ai.google.dev/gemma/docs/core/prompt-formatting-gemma4
struct gemma4_model_turn_builder {
json & messages;
size_t pos;
json tool_calls = json::array();
json tool_responses = json::array();
json content;
json reasoning_content;
#include <algorithm>
#include <cctype>
#include <map>
#include <optional>
gemma4_model_turn_builder(json & msgs, size_t pos) : messages(msgs), pos(pos) {}
namespace {
void collect() {
// Collect the first assistant message
auto & msg = messages[pos];
if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {
// According to the prompt formatting guide, we need to preserve reasoning_content
// between function calls. The current chat templates do not support this, but we will do it anyway.
reasoning_content = msg.at("reasoning_content");
using gemma4 = common_chat_peg_gemma4_mapper;
struct depth_guard {
int & depth;
depth_guard(int & depth) : depth(depth) {
if (++depth > COMMON_RECURSIVE_LIMIT) {
throw std::invalid_argument("JSON schema is nested more than " + std::to_string(COMMON_RECURSIVE_LIMIT) + " levels deep");
}
for (auto & tc : msg.at("tool_calls")) {
tool_calls.push_back(tc);
}
pos++;
}
~depth_guard() { depth--; }
};
// Collect tool call results
while (pos < messages.size() && messages[pos].value("role", "") == "tool") {
collect_result(messages[pos]);
pos++;
}
// Gemma 4 value parsers
void gemma4_value_rules(common_chat_peg_builder & p) {
// The tags are what gemma4_to_json() in the mapper reads, the rules may wrap them freely.
p.rule("gemma4-string", p.tag(gemma4::STRING, p.literal("<|\"|>") + p.tag(gemma4::STRING_CONTENT, p.until("<|\"|>")) + p.literal("<|\"|>")));
// Check if the next assistant message is the final message
if (pos < messages.size() && messages[pos].value("role", "") == "assistant") {
auto & next = messages[pos];
if (!has_tool_calls(next) && has_content(next)) {
content = next.at("content");
pos++;
}
p.rule("gemma4-bool", p.tag(gemma4::BOOL, p.json_bool()));
p.rule("gemma4-null", p.tag(gemma4::NULL_, p.json_null()));
p.rule("gemma4-number", p.tag(gemma4::NUMBER, p.json_number()));
// Keys are bare in the template, but Gemma 4 occasionally wraps them in quotes
p.rule("gemma4-dict-key", p.tag(gemma4::DICT_KEY, p.choice({
p.literal("<|\"|>") + p.tag(gemma4::DICT_KEY_NAME, p.until("<|\"|>")) + p.literal("<|\"|>"),
p.literal("\"") + p.tag(gemma4::DICT_KEY_NAME, p.chars("[^\"]", 1, -1)) + p.literal("\""),
p.literal("'") + p.tag(gemma4::DICT_KEY_NAME, p.chars("[^']", 1, -1)) + p.literal("'"),
p.tag(gemma4::DICT_KEY_NAME, p.chars("[^:}]", 1, -1)),
}) + p.literal(":")));
p.rule("gemma4-dict-kv", p.tag(gemma4::DICT_KV, p.ref("gemma4-dict-key") + p.space() + p.ref("gemma4-value")));
p.rule("gemma4-dict", [&]() {
auto ws = p.space();
auto member = p.ref("gemma4-dict-kv");
auto members = p.sequence({member, p.zero_or_more(p.sequence({p.literal(","), ws, member}))});
return p.tag(gemma4::DICT, p.sequence({
p.literal("{"), ws,
p.choice({p.literal("}"), p.sequence({members, ws, p.literal("}")})})
}));
});
p.rule("gemma4-array", [&]() {
auto ws = p.space();
auto value = p.ref("gemma4-value");
auto elements = p.sequence({value, p.zero_or_more(p.sequence({p.literal(","), ws, value}))});
return p.tag(gemma4::ARRAY, p.sequence({
p.literal("["), ws,
p.choice({p.literal("]"), p.sequence({elements, ws, p.literal("]")})})
}));
});
p.rule("gemma4-value", [&]() {
return p.choice({
p.ref("gemma4-string"), p.ref("gemma4-dict"), p.ref("gemma4-array"),
p.ref("gemma4-number"), p.ref("gemma4-bool"), p.ref("gemma4-null")
});
});
}
// Gemma 4 uses different delimiters and cannot leverage the json-schema-to-grammar implementation.
// This implements a subset to enforce properties.
class gemma4_schema_parser {
common_chat_peg_builder & p_;
std::string root_;
int depth_ = 0;
std::map<const common_chat_schema *, std::string> refs_;
public:
gemma4_schema_parser(common_chat_peg_builder & p, std::string root) : p_(p), root_(std::move(root)) {}
common_peg_parser visit(const common_chat_schema & node) { return visit(node, root_); }
private:
common_peg_parser visit(const common_chat_schema & node, const std::string & name) {
depth_guard guard(depth_);
switch (node.kind()) {
case common_chat_schema::KIND_REF: return visit_ref(node.as<common_chat_schema_ref>());
case common_chat_schema::KIND_ANY_OF: return visit_any_of(node.as<common_chat_schema_any_of>(), name);
case common_chat_schema::KIND_ALL_OF: return visit_all_of(node.as<common_chat_schema_all_of>(), name);
case common_chat_schema::KIND_ARRAY: return visit_array(node.as<common_chat_schema_array>(), name);
case common_chat_schema::KIND_TUPLE: return visit_tuple(node.as<common_chat_schema_tuple>(), name);
case common_chat_schema::KIND_OBJECT: return visit_object(node.as<common_chat_schema_object>(), name);
default: return p_.ref("gemma4-value");
}
}
void collect_result(const json & curr) {
json response;
if (curr.contains("content")) {
const auto & content = curr.at("content");
if (content.is_string()) {
// Try to parse the content as JSON; fall back to raw string
try {
response = json::parse(content.get<std::string>());
} catch (...) {
response = content;
// Whether a schema contributes a shape of its own, as opposed to accepting any value.
static bool shaped(const common_chat_schema & node) {
switch (node.kind()) {
case common_chat_schema::KIND_REF:
return node.as<common_chat_schema_ref>().target != nullptr;
case common_chat_schema::KIND_ANY_OF:
for (const auto & child : node.as<common_chat_schema_any_of>().children) {
if (shaped(*child)) {
return true;
}
}
return false;
case common_chat_schema::KIND_ALL_OF: {
const auto & children = node.as<common_chat_schema_all_of>().children;
return children.size() == 1 && shaped(*children[0]);
}
case common_chat_schema::KIND_ARRAY:
case common_chat_schema::KIND_TUPLE:
case common_chat_schema::KIND_OBJECT:
return true;
default:
return false;
}
}
// The template writes keys bare. Models keep that for identifiers and quote anything else,
// with single or double quotes or the string delimiter, so those are accepted as well.
common_peg_parser key(const std::string & key) {
auto name = [&]() { return p_.tag(gemma4::DICT_KEY_NAME, p_.literal(key)); };
bool identifier = !key.empty() && !std::isdigit(static_cast<unsigned char>(key[0]));
for (char c : key) {
if (!std::isalnum(static_cast<unsigned char>(c)) && c != '_') {
identifier = false;
}
}
if (identifier) {
return p_.tag(gemma4::DICT_KEY, name() + p_.literal(":"));
}
auto alts = p_.choice();
if (!key.empty() && key.find_first_of(":}") == std::string::npos && std::string("'\"<").find(key[0]) == std::string::npos) {
// Check if key is unambiguous
alts |= name();
}
// Gemma 4 may emit keys in its string delimiters or quotes
alts |= p_.literal("<|\"|>") + name() + p_.literal("<|\"|>");
if (key.find('\'') == std::string::npos) {
alts |= p_.literal("'") + name() + p_.literal("'");
}
if (key.find('"') == std::string::npos) {
alts |= p_.literal("\"") + name() + p_.literal("\"");
}
return p_.tag(gemma4::DICT_KEY, alts + p_.literal(":"));
}
common_peg_parser visit_ref(const common_chat_schema_ref & ref) {
if (!ref.target) {
return p_.ref("gemma4-value");
}
auto it = refs_.find(ref.target);
if (it != refs_.end()) {
return p_.ref(it->second);
}
// A target reached from several places gets one rule, named by the order it was first seen.
auto rule = root_ + "-ref-" + std::to_string(refs_.size());
refs_[ref.target] = rule;
return p_.rule(rule, [&]() { return visit(*ref.target, rule + "-value"); });
}
// Shaped alternatives are tried first, then anything else collapses into one generic value.
common_peg_parser visit_any_of(const common_chat_schema_any_of & node, const std::string & name) {
if (!shaped(node)) {
return p_.ref("gemma4-value");
}
if (node.children.size() == 1) {
return visit(*node.children[0], name);
}
auto alts = p_.choice();
bool loose = false;
for (size_t i = 0; i < node.children.size(); i++) {
if (shaped(*node.children[i])) {
alts |= visit(*node.children[i], name + "-" + std::to_string(i));
} else {
response = content;
loose = true;
}
}
std::string name;
// Match name with corresponding tool call
size_t idx = tool_responses.size();
if (idx < tool_calls.size()) {
auto & tc = tool_calls[idx];
if (tc.contains("function")) {
name = tc.at("function").value("name", "");
}
if (loose) {
alts |= p_.ref("gemma4-value");
}
// Fallback to the tool call id
if (name.empty()) {
name = curr.value("tool_call_id", "");
}
tool_responses.push_back({{"name", name}, {"response", response}});
return p_.rule(name, alts);
}
json build() {
collect();
common_peg_parser visit_all_of(const common_chat_schema_all_of & node, const std::string & name) {
if (node.children.size() == 1) {
return visit(*node.children[0], name);
}
return p_.ref("gemma4-value");
}
json msg = {
{"role", "assistant"},
{"tool_calls", tool_calls},
common_peg_parser visit_array(const common_chat_schema_array & node, const std::string & name) {
auto ws = p_.space();
auto item = node.items && shaped(*node.items) ? visit(*node.items, name + "-item") : p_.ref("gemma4-value");
auto elements = item + p_.zero_or_more(p_.literal(",") + ws + item);
return p_.rule(name, p_.tag(gemma4::ARRAY,
p_.literal("[") + ws + (p_.literal("]") | (elements + ws + p_.literal("]")))));
}
common_peg_parser visit_tuple(const common_chat_schema_tuple & node, const std::string & name) {
auto ws = p_.space();
auto items = p_.sequence();
for (size_t i = 0; i < node.items.size(); i++) {
if (i > 0) {
items = items + p_.literal(",") + ws;
}
items = items + visit(*node.items[i], name + "-" + std::to_string(i));
}
return p_.rule(name, p_.tag(gemma4::ARRAY, p_.literal("[") + ws + items + ws + p_.literal("]")));
}
common_peg_parser visit_object(const common_chat_schema_object & node, const std::string & name) {
auto ws = p_.space();
std::vector<const common_chat_schema_property *> props;
for (const auto & prop : node.properties) {
props.push_back(&prop);
}
// The Gemma 4 jinja template sorts properties, so our rules need to match.
auto lower = [](const std::string & s) {
std::string out = s;
for (auto & c : out) {
c = static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
}
return out;
};
if (!tool_responses.empty()) {
msg["tool_responses"] = tool_responses;
}
if (!content.is_null()) {
msg["content"] = content;
}
if (!reasoning_content.is_null()) {
msg["reasoning_content"] = reasoning_content;
}
return msg;
}
std::stable_sort(props.begin(), props.end(), [&](const auto * a, const auto * b) {
return lower(a->name) < lower(b->name);
});
static bool has_content(const json & msg) {
if (!msg.contains("content") || msg.at("content").is_null()) {
return false;
if (props.empty()) {
if (node.additional_properties) {
return p_.ref("gemma4-dict");
}
return p_.rule(name, p_.tag(gemma4::DICT, p_.literal("{") + ws + p_.literal("}")));
}
const auto & content = msg.at("content");
if (content.is_string() && !content.get<std::string>().empty()) {
return true;
}
if (content.is_array() && !content.empty()) {
return true;
}
return false;
}
static bool has_tool_calls(const json & msg) {
return msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty();
std::optional<common_peg_parser> rest;
std::optional<common_peg_parser> first;
if (node.additional_properties) {
auto kv = p_.ref("gemma4-dict-kv");
rest = p_.zero_or_more(p_.literal(",") + ws + kv);
first = p_.optional(kv + *rest);
}
auto then_rest = [&](const common_peg_parser & parser) {
return rest ? parser + *rest : parser;
};
for (size_t i = props.size(); i-- > 0;) {
const auto & prop = *props[i];
auto kv = p_.tag(gemma4::DICT_KV, key(prop.name) + ws + visit(*prop.schema, name + "-" + std::to_string(i)));
common_peg_parser rest_i = then_rest(p_.literal(",") + ws + kv);
common_peg_parser first_i = then_rest(kv);
if (!prop.required) {
rest_i = then_rest(p_.optional(p_.literal(",") + ws + kv));
first_i = first ? (first_i | *first) : p_.optional(first_i);
}
if (i > 0) {
rest = p_.rule(name + "-rest-" + std::to_string(i), rest_i);
}
first = first_i;
}
return p_.rule(name, p_.tag(gemma4::DICT, p_.literal("{") + ws + *first + ws + p_.literal("}")));
}
};
void convert_tool_responses_gemma4(json & messages) {
json result = json::array();
size_t i = 0;
while (i < messages.size()) {
auto & msg = messages[i];
if (msg.value("role", "") != "assistant" || !msg.contains("tool_calls") ||
!msg.at("tool_calls").is_array() || msg.at("tool_calls").empty()) {
result.push_back(msg);
i++;
continue;
}
gemma4_model_turn_builder builder(messages, i);
result.push_back(builder.build());
i = builder.pos;
}
messages = result;
}
}
} // namespace
common_chat_params common_chat_params_init_gemma4(const common_chat_template & tmpl,
const autoparser::generation_params & inputs) {
@@ -218,51 +350,29 @@ common_chat_params common_chat_params_init_gemma4(const common_chat_template &
}
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
// Gemma4 tool calling syntax
// Rules should match traversal logic in gemma4_to_json()
p.rule("gemma4-string-content", p.until("<|\"|>"));
p.rule("gemma4-string", p.literal("<|\"|>") + p.ref("gemma4-string-content") + p.literal("<|\"|>"));
p.rule("gemma4-bool", p.json_bool());
p.rule("gemma4-null", p.json_null());
p.rule("gemma4-number", p.json_number());
p.rule("gemma4-dict-key", p.rule("gemma4-dict-key-name", p.chars("[^:}]", 1, -1)) + p.literal(":"));
p.rule("gemma4-dict-kv", p.ref("gemma4-dict-key") + p.space() + p.ref("gemma4-value"));
p.rule("gemma4-dict", [&]() {
auto ws = p.space();
auto member = p.ref("gemma4-dict-kv");
auto members = p.sequence({member, p.zero_or_more(p.sequence({p.literal(","), ws, member}))});
return p.sequence({
p.literal("{"), ws,
p.choice({p.literal("}"), p.sequence({members, ws, p.literal("}")})})
});
});
p.rule("gemma4-array", [&]() {
auto ws = p.space();
auto value = p.ref("gemma4-value");
auto elements = p.sequence({value, p.zero_or_more(p.sequence({p.literal(","), ws, value}))});
return p.sequence({
p.literal("["), ws,
p.choice({p.literal("]"), p.sequence({elements, ws, p.literal("]")})})
});
});
p.rule("gemma4-value", [&]() {
return p.choice({
p.ref("gemma4-string"), p.ref("gemma4-dict"), p.ref("gemma4-array"),
p.ref("gemma4-number"), p.ref("gemma4-bool"), p.ref("gemma4-null")
});
});
gemma4_value_rules(p);
auto tool_choice = p.choice();
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
const auto & function = tool.at("function");
std::string name = function.at("name");
// TODO @aldehir : need to extend json-schema-to-grammar to produce more than JSON rules
// const auto & params = function.at("parameters");
// Arguments follow the shape of the parameter schema when it is an object,
// see gemma4_schema_parser, and are a generic dict otherwise.
auto args = p.ref("gemma4-dict");
try {
auto doc = common_chat_schema_from_json(common_chat_tool_parameters(function));
if (doc.root->kind() == common_chat_schema::KIND_OBJECT) {
args = gemma4_schema_parser(p, "tool-" + std::to_string(tool_index) + "-args").visit(*doc.root);
}
} catch (const std::exception & e) {
LOG_WRN("%s: tool %s uses the generic argument parser: %s\n", __func__, name.c_str(), e.what());
}
tool_choice |= p.rule("tool-" + std::to_string(tool_index), p.tool(p.sequence({
p.tool_open(p.tool_name(p.literal(name)) + p.peek(p.literal("{"))),
p.tool_args(p.ref("gemma4-dict")),
p.tool_args(args),
})));
});
-6
View File
@@ -41,12 +41,6 @@ std::string common_chat_template_generation_prompt_impl(
bool is_lfm2_template(const std::string & src);
namespace workaround {
void convert_tool_responses_gemma4(json & messages);
}
common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
@@ -1,282 +0,0 @@
{%- macro format_parameters(properties, required) -%}
{%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%}
{%- set ns = namespace(found_first=false) -%}
{%- for key, value in properties | dictsort -%}
{%- set add_comma = false -%}
{%- if key not in standard_keys -%}
{%- if ns.found_first %},{% endif -%}
{%- set ns.found_first = true -%}
{{ key }}:{
{%- if value['description'] -%}
description:<|"|>{{ value['description'] }}<|"|>
{%- set add_comma = true -%}
{%- endif -%}
{%- if value['nullable'] %}
{%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
nullable:true
{%- endif -%}
{%- if value['type'] | upper == 'STRING' -%}
{%- if value['enum'] -%}
{%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
enum:{{ format_argument(value['enum']) }}
{%- endif -%}
{%- elif value['type'] | upper == 'OBJECT' -%}
,properties:{
{%- if value['properties'] is defined and value['properties'] is mapping -%}
{{- format_parameters(value['properties'], value['required'] | default([])) -}}
{%- elif value is mapping -%}
{{- format_parameters(value, value['required'] | default([])) -}}
{%- endif -%}
}
{%- if value['required'] -%}
,required:[
{%- for item in value['required'] | default([]) -%}
<|"|>{{- item -}}<|"|>
{%- if not loop.last %},{% endif -%}
{%- endfor -%}
]
{%- endif -%}
{%- elif value['type'] | upper == 'ARRAY' -%}
{%- if value['items'] is mapping and value['items'] -%}
,items:{
{%- set ns_items = namespace(found_first=false) -%}
{%- for item_key, item_value in value['items'] | dictsort -%}
{%- if item_value is not none -%}
{%- if ns_items.found_first %},{% endif -%}
{%- set ns_items.found_first = true -%}
{%- if item_key == 'properties' -%}
properties:{
{%- if item_value is mapping -%}
{{- format_parameters(item_value, value['items']['required'] | default([])) -}}
{%- endif -%}
}
{%- elif item_key == 'required' -%}
required:[
{%- for req_item in item_value -%}
<|"|>{{- req_item -}}<|"|>
{%- if not loop.last %},{% endif -%}
{%- endfor -%}
]
{%- elif item_key == 'type' -%}
{%- if item_value is string -%}
type:{{ format_argument(item_value | upper) }}
{%- else -%}
type:{{ format_argument(item_value | map('upper') | list) }}
{%- endif -%}
{%- else -%}
{{ item_key }}:{{ format_argument(item_value) }}
{%- endif -%}
{%- endif -%}
{%- endfor -%}
}
{%- endif -%}
{%- endif -%}
{%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
type:<|"|>{{ value['type'] | upper }}<|"|>}
{%- endif -%}
{%- endfor -%}
{%- endmacro -%}
{%- macro format_function_declaration(tool_data) -%}
declaration:{{- tool_data['function']['name'] -}}{description:<|"|>{{- tool_data['function']['description'] -}}<|"|>
{%- set params = tool_data['function']['parameters'] -%}
{%- if params -%}
,parameters:{
{%- if params['properties'] -%}
properties:{ {{- format_parameters(params['properties'], params['required']) -}} },
{%- endif -%}
{%- if params['required'] -%}
required:[
{%- for item in params['required'] -%}
<|"|>{{- item -}}<|"|>
{{- ',' if not loop.last -}}
{%- endfor -%}
],
{%- endif -%}
{%- if params['type'] -%}
type:<|"|>{{- params['type'] | upper -}}<|"|>}
{%- endif -%}
{%- endif -%}
{%- if 'response' in tool_data['function'] -%}
{%- set response_declaration = tool_data['function']['response'] -%}
,response:{
{%- if response_declaration['description'] -%}
description:<|"|>{{- response_declaration['description'] -}}<|"|>,
{%- endif -%}
{%- if response_declaration['type'] | upper == 'OBJECT' -%}
type:<|"|>{{- response_declaration['type'] | upper -}}<|"|>}
{%- endif -%}
{%- endif -%}
}
{%- endmacro -%}
{%- macro format_argument(argument, escape_keys=True) -%}
{%- if argument is string -%}
{{- '<|"|>' + argument + '<|"|>' -}}
{%- elif argument is boolean -%}
{{- 'true' if argument else 'false' -}}
{%- elif argument is mapping -%}
{{- '{' -}}
{%- set ns = namespace(found_first=false) -%}
{%- for key, value in argument | dictsort -%}
{%- if ns.found_first %},{% endif -%}
{%- set ns.found_first = true -%}
{%- if escape_keys -%}
{{- '<|"|>' + key + '<|"|>' -}}
{%- else -%}
{{- key -}}
{%- endif -%}
:{{- format_argument(value, escape_keys=escape_keys) -}}
{%- endfor -%}
{{- '}' -}}
{%- elif argument is sequence -%}
{{- '[' -}}
{%- for item in argument -%}
{{- format_argument(item, escape_keys=escape_keys) -}}
{%- if not loop.last %},{% endif -%}
{%- endfor -%}
{{- ']' -}}
{%- else -%}
{{- argument -}}
{%- endif -%}
{%- endmacro -%}
{%- macro strip_thinking(text) -%}
{%- set ns = namespace(result='') -%}
{%- for part in text.split('<channel|>') -%}
{%- if '<|channel>' in part -%}
{%- set ns.result = ns.result + part.split('<|channel>')[0] -%}
{%- else -%}
{%- set ns.result = ns.result + part -%}
{%- endif -%}
{%- endfor -%}
{{- ns.result | trim -}}
{%- endmacro -%}
{%- set ns = namespace(prev_message_type=None, last_user_message=-1) -%}
{%- set loop_messages = messages -%}
{{- bos_token -}}
{#- Handle System/Tool Definitions Block -#}
{%- if (enable_thinking is defined and enable_thinking) or tools or messages[0]['role'] in ['system', 'developer'] -%}
{{- '<|turn>system\n' -}}
{#- Inject Thinking token at the very top of the FIRST system turn -#}
{%- if enable_thinking is defined and enable_thinking -%}
{{- '<|think|>\n' -}}
{%- set ns.prev_message_type = 'think' -%}
{%- endif -%}
{%- if messages[0]['role'] in ['system', 'developer'] -%}
{{- messages[0]['content'] | trim -}}
{%- set loop_messages = messages[1:] -%}
{%- endif -%}
{%- if tools -%}
{%- for tool in tools %}
{{- '<|tool>' -}}
{{- format_function_declaration(tool) | trim -}}
{{- '<tool|>' -}}
{%- endfor %}
{%- set ns.prev_message_type = 'tool' -%}
{%- endif -%}
{{- '<turn|>\n' -}}
{%- endif %}
{#- Find last user message -#}
{%- for message in loop_messages -%}
{%- if message['role'] == 'user' -%}
{%- set ns.last_user_message = loop.index0 -%}
{%- endif -%}
{%- endfor -%}
{#- Loop through messages -#}
{%- for message in loop_messages -%}
{%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}
{%- if not (ns.prev_message_type == 'tool_response' and message['tool_calls']) -%}
{{- '<|turn>' + role + '\n' }}
{%- endif -%}
{%- set ns.prev_message_type = None -%}
{%- if message['tool_calls'] -%}
{#- Preserve reasoning between tool calls for model turns that come after the last user turn -#}
{%- if message['reasoning_content'] and loop.index0 > ns.last_user_message -%}
{{- '<|channel>thought\n' -}}
{{- message['reasoning_content'] -}}
{{- '<channel|>' -}}
{%- endif -%}
{%- for tool_call in message['tool_calls'] -%}
{%- set function = tool_call['function'] -%}
{{- '<|tool_call>call:' + function['name'] + '{' -}}
{%- if function['arguments'] is mapping -%}
{%- set ns_args = namespace(found_first=false) -%}
{%- for key, value in function['arguments'] | dictsort -%}
{%- if ns_args.found_first %},{% endif -%}
{%- set ns_args.found_first = true -%}
{{- key -}}:{{- format_argument(value, escape_keys=False) -}}
{%- endfor -%}
{%- elif function['arguments'] is string -%}
{{- function['arguments'] -}}
{%- endif -%}
{{- '}<tool_call|>' -}}
{%- endfor -%}
{%- set ns.prev_message_type = 'tool_call' -%}
{%- endif -%}
{%- if message['tool_responses'] -%}
{#- Tool Response handling -#}
{%- for tool_response in message['tool_responses'] -%}
{{- '<|tool_response>' -}}
{%- if tool_response['response'] is mapping -%}
{{- 'response:' + tool_response['name'] | default('unknown') + '{' -}}
{%- for key, value in tool_response['response'] | dictsort -%}
{{- key -}}:{{- format_argument(value, escape_keys=False) -}}
{%- if not loop.last %},{% endif -%}
{%- endfor -%}
{{- '}' -}}
{%- else -%}
{{- 'response:' + tool_response['name'] | default('unknown') + '{value:' + format_argument(tool_response['response'], escape_keys=False) + '}' -}}
{%- endif -%}
{{- '<tool_response|>' -}}
{%- endfor -%}
{%- set ns.prev_message_type = 'tool_response' -%}
{%- endif -%}
{%- if message['content'] is string -%}
{%- if role == 'model' -%}
{{- strip_thinking(message['content']) -}}
{%- else -%}
{{- message['content'] | trim -}}
{%- endif -%}
{%- elif message['content'] is sequence -%}
{%- for item in message['content'] -%}
{%- if item['type'] == 'text' -%}
{%- if role == 'model' -%}
{{- strip_thinking(item['text']) -}}
{%- else -%}
{{- item['text'] | trim -}}
{%- endif -%}
{%- elif item['type'] == 'image' -%}
{{- '<|image|>' -}}
{%- set ns.prev_message_type = 'image' -%}
{%- elif item['type'] == 'audio' -%}
{{- '<|audio|>' -}}
{%- set ns.prev_message_type = 'audio' -%}
{%- elif item['type'] == 'video' -%}
{{- '<|video|>' -}}
{%- set ns.prev_message_type = 'video' -%}
{%- endif -%}
{%- endfor -%}
{%- endif -%}
{%- if not (message['tool_responses'] and not message['content']) -%}
{{- '<turn|>\n' -}}
{%- endif -%}
{%- endfor -%}
{%- if add_generation_prompt -%}
{%- if ns.prev_message_type != 'tool_response' -%}
{{- '<|turn>model\n' -}}
{%- endif -%}
{%- if not enable_thinking | default(false) -%}
{{- '<|channel>thought\n<channel|>' -}}
{%- endif -%}
{%- endif -%}
+55
View File
@@ -3267,6 +3267,61 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
.expect(message_with_tool_calls("amount", R"({"orig": 1.5e10})"))
.run();
// The argument grammar follows the shape of the parameter schema: keys in
// alphabetical order, optional keys skipped in place, nested objects and arrays
// shaped the same way, any other value accepted as written.
common_chat_tool shape_tool{
/* .name = */ "shape",
/* .description = */ "Schema-shaped arguments",
/* .parameters = */ R"({
"type": "object",
"properties": {
"path": {"type": "string"},
"edits": {
"type": "array",
"items": {
"type": "object",
"properties": {
"oldText": {"type": "string"},
"newText": {"type": "string"}
},
"required": ["oldText", "newText"]
}
},
"mode": {"type": "string", "enum": ["fast", "safe"]},
"count": {"type": "integer"},
"headers": {
"type": "object",
"properties": {
"content-type": {"type": "string"},
"accept:lang": {"type": "string"}
}
},
"options": {"type": "object"}
},
"required": ["path", "edits"]
})",
};
tst.test(
"<|tool_call>call:shape{edits:[{newText:<|\"|>b<|\"|>,oldText:<|\"|>a<|\"|>}],path:<|\"|>f.c<|\"|>}<tool_call|>")
.tools({ shape_tool })
.expect(message_with_tool_calls("shape", R"({"edits":[{"newText":"b","oldText":"a"}],"path":"f.c"})"))
.run();
// Optional keys in place, quoted keys that are not identifiers
tst.test(
"<|tool_call>call:shape{count:3,edits:[],headers:{<|\"|>accept:lang<|\"|>:<|\"|>en<|\"|>,'content-type':<|\"|>text<|\"|>},mode:<|\"|>fast<|\"|>,options:{b:1,a:[true]},path:<|\"|>f<|\"|>}<tool_call|>")
.tools({ shape_tool })
.expect(message_with_tool_calls("shape", R"({"count":3,"edits":[],"headers":{"accept:lang":"en","content-type":"text"},"mode":"fast","options":{"b":1,"a":[true]},"path":"f"})"))
.run();
tst.test(
"<|tool_call>call:shape{edits:[],headers:{\"content-type\":<|\"|>text<|\"|>},path:<|\"|>f<|\"|>}<tool_call|>")
.tools({ shape_tool })
.expect(message_with_tool_calls("shape", R"({"edits":[],"headers":{"content-type":"text"},"path":"f"})"))
.run();
// Edge cases
tst.test(
"<|channel>thought\n<channel|>Hello, world!\nWhat's up?<channel|>")