Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix DeepSeek4 crafted template (#25414)

Browse Source 
* chat: fix DS4 template to explicitly follow reference behavior

* Support DeepSeekv4 flag (`drop_reasoning`).

* fix: hook DS3.2 parser for DS4 as well

* fix: add tool result reordering

* fix: post-merge
This commit is contained in:
Piotr Wilkin (ilintar)
2026-07-22 12:54:40 +02:00
committed by GitHub
parent 3ce7da2c85
commit f534da26e4
4 changed files with 382 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/chat.cpp
+95 -10
View File
@@ -15,11 +15,13 @@
#include "nlohmann/json.hpp"
#include <algorithm>
#include <cstdio>
#include <cstdlib>
#include <ctime>
#include <exception>
#include <functional>
#include <map>
#include <optional>
#include <sstream>
@@ -1855,12 +1857,89 @@ static common_chat_params common_chat_params_init_gigachat_v3(
return data;
}
// The DeepSeek V4 reference implementation renders consecutive tool results into a single
// user block, ordered by the tool call order of the preceding assistant message (matched
// by tool call id) rather than by the order they appear in the conversation.
static json deepseek_v4_sort_tool_results(const json & messages) {
json adjusted = messages;
std::map<std::string, size_t> call_order;
for (size_t i = 0; i < adjusted.size();) {
const auto & msg = adjusted[i];
const auto role = msg.value("role", "");
if (role == "assistant" && msg.contains("tool_calls") &&
msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) {
call_order.clear();
const auto & tool_calls = msg.at("tool_calls");
for (size_t idx = 0; idx < tool_calls.size(); idx++) {
auto id = tool_calls[idx].value("id", "");
if (!id.empty()) {
call_order[id] = idx;
}
}
i++;
continue;
}
if (role != "user" && role != "tool") {
i++;
continue;
}
// collect a maximal run of user/tool messages - they render into one user block
std::vector<size_t> tool_positions;
size_t run_end = i;
for (; run_end < adjusted.size(); run_end++) {
const auto r = adjusted[run_end].value("role", "");
if (r == "tool") {
tool_positions.push_back(run_end);
} else if (r != "user") {
break;
}
}
if (tool_positions.size() > 1 && !call_order.empty()) {
std::vector<json> results;
results.reserve(tool_positions.size());
for (auto pos : tool_positions) {
results.push_back(adjusted[pos]);
}
std::stable_sort(results.begin(), results.end(), [&](const json & a, const json & b) {
const auto order = [&](const json & m) {
auto it = call_order.find(m.value("tool_call_id", ""));
return it == call_order.end() ? (size_t) 0 : it->second;
};
return order(a) < order(b);
});
for (size_t k = 0; k < tool_positions.size(); k++) {
adjusted[tool_positions[k]] = std::move(results[k]);
}
}
i = run_end;
}
return adjusted;
}
static common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl,
const autoparser::generation_params & inputs) {
common_chat_params data;
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
// V4 uses the same DSML markup as V3.2, but names the tool call block "tool_calls"
// instead of "function_calls", renders tool results in tool call order and its
// non-thinking generation prompt ends with a bare </think> instead of an empty
// <think></think> pair.
const bool is_v4 = tmpl.source().find("function_calls") == std::string::npos;
std::optional<json> adjusted_messages;
if (is_v4) {
adjusted_messages = deepseek_v4_sort_tool_results(inputs.messages);
}
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, adjusted_messages);
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, adjusted_messages);
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
data.supports_thinking = true;
data.thinking_start_tag = "<think>";
@@ -1879,8 +1958,9 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
const std::string DSML = "｜DSML｜";
const std::string THINK_START = "<think>";
const std::string THINK_END = "</think>";
const std::string FC_START = "<" + DSML + "function_calls>";
const std::string FC_END = "</" + DSML + "function_calls>";
const std::string TC_BLOCK = is_v4 ? "tool_calls" : "function_calls";
const std::string FC_START = "<" + DSML + TC_BLOCK + ">";
const std::string FC_END = "</" + DSML + TC_BLOCK + ">";
const std::string INVOKE_START = "<" + DSML + "invoke";
const std::string INVOKE_END = "</" + DSML + "invoke>";
const std::string PARAM_START = "<" + DSML + "parameter";
@@ -1907,8 +1987,11 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);
} else if (extract_reasoning) {
// Thinking disabled but reasoning extraction requested: the generation prompt
// contains an empty <think></think> pair that must still be consumed.
reasoning = p.optional(p.literal(THINK_START) + p.until(THINK_END) + p.literal(THINK_END));
// contains an empty <think></think> pair (V3.2) or a bare </think> (V4) that
// must still be consumed.
reasoning = is_v4
? p.optional(p.literal(THINK_END))
: p.optional(p.literal(THINK_START) + p.until(THINK_END) + p.literal(THINK_END));
}
if (has_response_format) {
@@ -2612,12 +2695,14 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
return common_chat_params_init_gigachat_v3(tmpl, params);
}
// DeepSeek V3.2 format detection: template defines dsml_token and uses it for tool calls.
// DeepSeek V3.2/V4 format detection: template defines dsml_token and uses it for tool calls.
// The template source contains the token as a variable assignment, not as a literal in markup.
// V3.2 names the tool call block "function_calls", V4 names it "tool_calls".
if (src.find("dsml_token") != std::string::npos &&
src.find("function_calls") != std::string::npos &&
src.find("DSML") != std::string::npos) {
LOG_DBG("Using specialized template: DeepSeek V3.2\n");
src.find("DSML") != std::string::npos &&
(src.find("function_calls") != std::string::npos ||
src.find("tool_calls") != std::string::npos)) {
LOG_DBG("Using specialized template: DeepSeek V3.2/V4\n");
return common_chat_params_init_deepseek_v3_2(tmpl, params);
}