model: DSV4.1 flash (#31086)

* model : add DeepSeek-V4.1-Flash (deepseek41)

* deepseek41 : remove MXFP8, convert the Engram table to BF16

* deepseek41 : compute the compressed-KV read layers once in the cache

* deepseek41 : use get_cells() for the shared-history check, drop llama_kv_cache::seq_pos_tok_le

* deepseek41 : gather the Engram rows in-graph like qwen4exp, drop the host gather

* deepseek41 : keep the Engram hash constants in hparams.ple_*, one module per Engram layer

* deepseek41 : drop the LLAMA_DSV41_KV_MIRROR toggle

* deepseek41 : keep il last in build_hc_mixes

* deepseek41 : sort the top-k rows before the sparse gather

* tests : add deepseek41 to the Metal fusion baseline

---------

Co-authored-by: Daniel Han <danielhanchen@gmail.com>
This commit is contained in:
Aman GuptaandDaniel Han authored and GitHub committed 2026-10-25 09:43:51 +08:00
1 parent d327bf1f07
commit 74bca39500
39 files changed
+2798 -104

No files matched your search

+260 -3
View File
@@ -81,6 +81,7 @@ void llama_model_dflash::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, hparams.dsv4_hc_sinkhorn_iters);
ml.get_key(LLM_KV_HYPER_CONNECTION_EPSILON, hparams.dsv4_hc_eps);
ml.get_arr(LLM_KV_ATTENTION_COMPRESS_RATIOS, hparams.dsv4_compress_ratios, false);
ml.get_key(LLM_KV_HYPER_CONNECTION_SINGLE_PASS, hparams.dsv4_hc_single_pass, false);
GGML_ASSERT(hparams.dsv4_o_group_count > 0); // avoid div by zero
@@ -94,6 +95,11 @@ void llama_model_dflash::load_arch_hparams(llama_model_loader & ml) {
}
GGML_ASSERT(hparams.n_swa > 0);
if (hparams.dsv4_hc_single_pass) {
// DeepSeek-V4.1 DSpark: block queries see the main-KV ring plus the block via the mask row of the
// first query, so that row's SWA window must reach window_size positions before the anchor
hparams.n_swa += 1;
}
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
hparams.set_swa_pattern(0);
for (uint32_t il = 0; il < hparams.n_layer_all; ++il) {
@@ -257,9 +263,12 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) {
const int64_t hc_dim = hc_mult * n_embd;
const int64_t hc_mix_dim = (2 + hc_mult) * hc_mult;
hc_head_fn = create_tensor(tn(LLM_TENSOR_HC_HEAD_FN, "weight"), {hc_dim, hc_mult}, 0);
hc_head_base = create_tensor(tn(LLM_TENSOR_HC_HEAD_BASE, "weight"), {hc_mult}, 0);
hc_head_scale = create_tensor(tn(LLM_TENSOR_HC_HEAD_SCALE, "weight"), {1}, 0);
// single-pass mHC: collapse with the last FFN pre-mix, no hc_head
if (!hparams.dsv4_hc_single_pass) {
hc_head_fn = create_tensor(tn(LLM_TENSOR_HC_HEAD_FN, "weight"), {hc_dim, hc_mult}, 0);
hc_head_base = create_tensor(tn(LLM_TENSOR_HC_HEAD_BASE, "weight"), {hc_mult}, 0);
hc_head_scale = create_tensor(tn(LLM_TENSOR_HC_HEAD_SCALE, "weight"), {1}, 0);
}
for (int i = 0; i < n_layer; ++i) {
auto & layer = layers[i];
@@ -1280,12 +1289,260 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_
}
}
// DeepSeek-V4.1 DSpark decoder (mtp.0..2), dual-mode by batch type like graph_dsv4:
// * embd batch -> writes every stage's kv_norm(wkv(main_proj(target features))) into its window ring
// * token batch -> [anchor, noise...] block through the stages, single-pass mHC, no per-head q norm
llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const llm_graph_params & params) :
llama_model_deepseek41::graph(params) {
const int64_t n_embd_inp = hparams.n_embd_inp_enc();
const int64_t n_embd_head = hparams.n_embd_head_k();
const int64_t n_embd_head_rope = hparams.n_rot();
const int64_t n_embd_head_nope = n_embd_head - n_embd_head_rope;
GGML_ASSERT(hparams.dsv4_hc_single_pass);
ggml_tensor * inp_pos = build_inp_pos();
llm_graph_input_attn_k_iswa * inp_attn = build_attn_inp_k_iswa();
// ratio-0 layers: plain rope at rope_theta, no YaRN (original_seq_len 0)
const auto rope = [&](ggml_tensor * t) {
t = ggml_rope_ext(ctx0, t, inp_pos, nullptr, n_embd_head_rope, rope_type, 0,
freq_base, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
return ggml_rope_set_offset(t, n_embd_head_nope);
};
ASSERT_EMBD_OR_TOKEN(ubatch);
if (ubatch.embd) {
auto inp = std::make_unique<llm_graph_input_embd>(n_embd_inp);
inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd_inp, n_tokens);
ggml_set_input(inp->embd);
ggml_tensor * inp_target = inp->embd;
cb(inp_target, "inp_target_features", -1);
res->add_input(std::move(inp));
ggml_tensor * inp_g = build_lora_mm(model.fc, inp_target, model.fc_s);
inp_g = build_norm(inp_g, model.output_norm_enc, nullptr, LLM_NORM_RMS, -1);
cb(inp_g, "main_x", -1);
for (int il = 0; il < n_layer; ++il) {
const auto & layer = model.layers[il];
ggml_tensor * kv = build_lora_mm(layer.wkv, inp_g);
kv = build_norm(kv, layer.attn_kv_norm, nullptr, LLM_NORM_RMS, il);
kv = ggml_reshape_3d(ctx0, kv, n_embd_head, 1, n_tokens);
kv = rope(kv);
cb(kv, "kv_injected", il);
if (inp_attn->self_k_rot_swa) {
kv = llama_mul_mat_hadamard(ctx0, kv, inp_attn->self_k_rot_swa);
}
ggml_build_forward_expand(gf, inp_attn->mctx->get_swa()->cpy_k(ctx0, kv, inp_attn->get_k_idxs_swa(), il));
}
res->t_embd = inp_g;
ggml_build_forward_expand(gf, inp_g);
return;
}
auto * tok_embd = model.tok_embd;
if (tok_embd == nullptr) {
GGML_ASSERT(cparams.ctx_other != nullptr);
const auto * model_other = llama_get_model(cparams.ctx_other);
GGML_ASSERT(model_other->tok_embd != nullptr && "DSpark decoder requires the target model's token embeddings");
tok_embd = model_other->tok_embd;
}
auto inp = std::make_unique<llm_graph_input_embd>(n_embd);
inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
ggml_set_input(inp->tokens);
ggml_tensor * inp_tokens = inp->tokens;
ggml_tensor * inpL = ggml_get_rows(ctx0, tok_embd, inp->tokens);
cb(inpL, "inp_noise_embd", -1);
res->add_input(std::move(inp));
const int64_t hc = hparams.dsv4_hc_mult;
const int64_t n_groups = hparams.dsv4_o_group_count;
const int64_t n_heads_group = n_head / n_groups;
const int64_t o_lora_rank = hparams.dsv4_o_lora_rank;
const int64_t o_group_dim = n_heads_group*n_embd_head;
const float kq_scale = 1.0f/sqrtf(float(n_embd_head));
GGML_ASSERT(n_embd_head == n_embd_head_v);
GGML_ASSERT(n_head % n_groups == 0);
inpL = ggml_reshape_3d(ctx0, inpL, n_embd, 1, n_tokens);
inpL = ggml_repeat_4d(ctx0, inpL, n_embd, hc, n_tokens, 1);
cb(inpL, "hc_init", -1);
// window of a block = window of its first query, see n_swa
{
ggml_tensor * m = inp_attn->self_kq_mask_swa_cnv;
const int64_t n_blocks = ubatch.n_seqs_unq;
const int64_t bd = n_blocks > 0 ? n_tokens / n_blocks : 0;
if (m && bd > 1 && n_tokens % n_blocks == 0 && m->ne[2] == 1 && m->ne[1] % bd == 0 && ggml_is_contiguous(m)) {
ggml_tensor * m4 = ggml_reshape_4d(ctx0, m, m->ne[0], bd, m->ne[1]/bd, m->ne[3]);
ggml_tensor * r0 = ggml_view_4d(ctx0, m4, m4->ne[0], 1, m4->ne[2], m4->ne[3], m4->nb[1], m4->nb[2], m4->nb[3], 0);
m4 = ggml_repeat(ctx0, r0, m4);
inp_attn->self_kq_mask_swa_cnv = ggml_reshape_4d(ctx0, m4, m->ne[0], m->ne[1], 1, m->ne[3]);
cb(inp_attn->self_kq_mask_swa_cnv, "dspark_block_mask", -1);
}
}
ggml_tensor * pre_prev = nullptr;
for (int il = 0; il < n_layer; ++il) {
const auto & layer = model.layers[il];
ggml_tensor * residual = inpL;
ggml_tensor * post = nullptr;
ggml_tensor * comb = nullptr;
ggml_tensor * attn_pre = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, false, il);
ggml_tensor * cur;
if (pre_prev == nullptr) {
// identity pre-mix of the first stage: copy 0
cur = ggml_cont(ctx0, ggml_view_2d(ctx0, inpL, n_embd, n_tokens, inpL->nb[2], 0));
} else {
cur = build_hc_pre(inpL, pre_prev, il);
}
cb(cur, "hc_attn_pre", il);
cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
cb(cur, "attn_norm", il);
// non-causal draft context: block queries see the main-KV ring plus every block key
{
ggml_tensor * qr = build_lora_mm(layer.wq_a, cur);
qr = build_norm(qr, layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il);
cb(qr, "qr_norm", il);
ggml_tensor * q = build_lora_mm(layer.wq_b, qr);
q = ggml_reshape_3d(ctx0, q, n_embd_head, n_head, n_tokens);
q = rope(q);
cb(q, "q", il);
ggml_tensor * kv = build_lora_mm(layer.wkv, cur);
kv = build_norm(kv, layer.attn_kv_norm, nullptr, LLM_NORM_RMS, il);
kv = ggml_reshape_3d(ctx0, kv, n_embd_head, 1, n_tokens);
kv = rope(kv);
cb(kv, "kv", il);
ggml_tensor * out = build_attn(inp_attn,
nullptr, nullptr, nullptr,
q, kv, kv,
nullptr, layer.attn_sinks, nullptr,
kq_scale, il);
cb(out, "attn_raw", il);
out = ggml_reshape_3d(ctx0, out, n_embd_head, n_head, n_tokens);
out = ggml_rope_ext_back(ctx0, out, inp_pos, nullptr, n_embd_head_rope, rope_type, 0,
freq_base, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
out = ggml_rope_set_offset(out, n_embd_head_nope);
cb(out, "attn_derope", il);
out = ggml_reshape_3d(ctx0, out, o_group_dim, n_groups, n_tokens);
out = ggml_permute(ctx0, out, 0, 2, 1, 3);
ggml_tensor * oa = ggml_mul_mat(ctx0, layer.wo_a, out);
cb(oa, "attn_wo_a", il);
oa = ggml_permute(ctx0, oa, 0, 2, 1, 3);
oa = ggml_cont_2d(ctx0, oa, o_lora_rank*n_groups, n_tokens);
cur = build_lora_mm(layer.wo_b, oa);
cb(cur, "attn_out", il);
}
inpL = build_hc_post(cur, residual, post, comb, il);
cb(inpL, "hc_attn_post", il);
residual = inpL;
ggml_tensor * ffn_pre = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, false, il);
cur = build_hc_pre(inpL, attn_pre, il);
cb(cur, "hc_ffn_pre", il);
cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
cb(cur, "ffn_norm", il);
ggml_tensor * moe_out = build_moe_ffn(cur,
layer.ffn_gate_inp,
layer.ffn_up_exps,
layer.ffn_gate_exps,
layer.ffn_down_exps,
layer.ffn_exp_probs_b,
n_expert, hparams.n_expert_used(),
LLM_FFN_SILU, hparams.expert_weights_norm,
hparams.expert_weights_scale,
(llama_expert_gating_func_type) hparams.expert_gating_func,
il);
cb(moe_out, "ffn_moe_out", il);
ggml_tensor * ffn_shexp = build_ffn(cur,
layer.ffn_up_shexp, nullptr, nullptr,
layer.ffn_gate_shexp, nullptr, nullptr,
layer.ffn_down_shexp, nullptr, nullptr,
nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il);
cb(ffn_shexp, "ffn_shexp", il);
cur = ggml_add(ctx0, moe_out, ffn_shexp);
cb(cur, "ffn_out", il);
inpL = build_hc_post(cur, residual, post, comb, il);
ggml_tensor * l_out = ggml_reshape_2d(ctx0, inpL, n_embd*hc, n_tokens);
cb(l_out, "l_out", il);
ggml_build_forward_expand(gf, l_out);
pre_prev = ffn_pre;
}
// no hc_head: collapse with the last FFN's pre-mix
ggml_tensor * cur = build_hc_pre(inpL, pre_prev, -1);
cb(cur, "hc_out", -1);
// confidence head takes the un-normed collapsed state
res->t_embd = cur;
cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1);
cb(cur, "result_norm", -1);
auto * output = model.output;
auto * output_s = model.output_s;
if (output == nullptr) {
GGML_ASSERT(cparams.ctx_other != nullptr);
const auto * model_other = llama_get_model(cparams.ctx_other);
GGML_ASSERT(model_other->output != nullptr && "DSpark decoder requires the target model's output projection");
output = model_other->output;
output_s = model_other->output_s;
}
cur = build_lora_mm(output, cur, output_s);
cb(cur, "result_output", -1);
res->t_logits = cur;
if (model.dspark_markov_w1) {
build_dspark_markov_head(*this, model, inp_tokens);
}
}
std::unique_ptr<llm_graph_context> llama_model_dflash::build_arch_graph(const llm_graph_params & params) const {
switch (params.gtype) {
case LLM_GRAPH_TYPE_ENCODER:
return std::make_unique<graph<true>>(*this, params);
case LLM_GRAPH_TYPE_DEFAULT:
case LLM_GRAPH_TYPE_DECODER:
if (hparams.dsv4_hc_mult > 0 && hparams.dsv4_hc_single_pass) {
return std::make_unique<graph_dsv41>(*this, params);
}
if (hparams.dsv4_hc_mult > 0) {
return std::make_unique<graph_dsv4>(*this, params);
}