From 372dc5ca157276beb6c67b2e92477e8d662e964a Mon Sep 17 00:00:00 2001 From: WuXin Date: Sun, 8 Nov 2026 09:36:15 +0530 Subject: [PATCH] common : sanitize invalid UTF-8 in PEG tool call arguments (#31861) --- common/chat-peg-parser.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 74b81dcdd..21620a7a4 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -348,7 +348,8 @@ void common_chat_peg_mapper::map(const common_peg_ast_node & node) { } if (is_tool_id && current_tool) { - auto text = trim_trailing_space(node.text); + auto sanitized = node.sanitized_text(); + auto text = trim_trailing_space(sanitized); if (text.size() >= 2 && text.front() == '"' && text.back() == '"') { text = text.substr(1, text.size() - 2); } @@ -401,7 +402,7 @@ void common_chat_peg_mapper::map(const common_peg_ast_node & node) { } if ((is_arg_value || is_arg_string_value) && current_tool) { - std::string value_content = std::string(node.text); + std::string value_content = node.sanitized_text(); std::string value_to_add; if (value_content.empty() && is_arg_string_value) { @@ -978,7 +979,7 @@ static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg } if (node.rule == "gemma4-string-content") { - return escape_json_string_inner(std::string(node.text)); + return escape_json_string_inner(node.sanitized_text()); } if (node.rule == "gemma4-string") { @@ -1195,7 +1196,7 @@ static std::string minimax_m3_value_to_json(const common_peg_ast_arena & arena, } if (node.tag == common_chat_peg_builder::TOOL_ARG_STRING_VALUE) { - return "\"" + escape_json_string_inner(std::string(node.text)) + (closed ? "\"" : ""); + return "\"" + escape_json_string_inner(node.sanitized_text()) + (closed ? "\"" : ""); } // Numbers and booleans are written verbatim by the template