model: DSV4.1 flash (#31086)
* model : add DeepSeek-V4.1-Flash (deepseek41) * deepseek41 : remove MXFP8, convert the Engram table to BF16 * deepseek41 : compute the compressed-KV read layers once in the cache * deepseek41 : use get_cells() for the shared-history check, drop llama_kv_cache::seq_pos_tok_le * deepseek41 : gather the Engram rows in-graph like qwen4exp, drop the host gather * deepseek41 : keep the Engram hash constants in hparams.ple_*, one module per Engram layer * deepseek41 : drop the LLAMA_DSV41_KV_MIRROR toggle * deepseek41 : keep il last in build_hc_mixes * deepseek41 : sort the top-k rows before the sparse gather * tests : add deepseek41 to the Metal fusion baseline --------- Co-authored-by: Daniel Han <danielhanchen@gmail.com>
This commit is contained in:
1 parent
d327bf1f07
commit
74bca39500
39 files changed
+2798
-104
No files matched your search
+260
-3
@@ -81,6 +81,7 @@ void llama_model_dflash::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, hparams.dsv4_hc_sinkhorn_iters);
|
||||
ml.get_key(LLM_KV_HYPER_CONNECTION_EPSILON, hparams.dsv4_hc_eps);
|
||||
ml.get_arr(LLM_KV_ATTENTION_COMPRESS_RATIOS, hparams.dsv4_compress_ratios, false);
|
||||
ml.get_key(LLM_KV_HYPER_CONNECTION_SINGLE_PASS, hparams.dsv4_hc_single_pass, false);
|
||||
|
||||
GGML_ASSERT(hparams.dsv4_o_group_count > 0); // avoid div by zero
|
||||
|
||||
@@ -94,6 +95,11 @@ void llama_model_dflash::load_arch_hparams(llama_model_loader & ml) {
|
||||
}
|
||||
|
||||
GGML_ASSERT(hparams.n_swa > 0);
|
||||
if (hparams.dsv4_hc_single_pass) {
|
||||
// DeepSeek-V4.1 DSpark: block queries see the main-KV ring plus the block via the mask row of the
|
||||
// first query, so that row's SWA window must reach window_size positions before the anchor
|
||||
hparams.n_swa += 1;
|
||||
}
|
||||
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
|
||||
hparams.set_swa_pattern(0);
|
||||
for (uint32_t il = 0; il < hparams.n_layer_all; ++il) {
|
||||
@@ -257,9 +263,12 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) {
|
||||
const int64_t hc_dim = hc_mult * n_embd;
|
||||
const int64_t hc_mix_dim = (2 + hc_mult) * hc_mult;
|
||||
|
||||
hc_head_fn = create_tensor(tn(LLM_TENSOR_HC_HEAD_FN, "weight"), {hc_dim, hc_mult}, 0);
|
||||
hc_head_base = create_tensor(tn(LLM_TENSOR_HC_HEAD_BASE, "weight"), {hc_mult}, 0);
|
||||
hc_head_scale = create_tensor(tn(LLM_TENSOR_HC_HEAD_SCALE, "weight"), {1}, 0);
|
||||
// single-pass mHC: collapse with the last FFN pre-mix, no hc_head
|
||||
if (!hparams.dsv4_hc_single_pass) {
|
||||
hc_head_fn = create_tensor(tn(LLM_TENSOR_HC_HEAD_FN, "weight"), {hc_dim, hc_mult}, 0);
|
||||
hc_head_base = create_tensor(tn(LLM_TENSOR_HC_HEAD_BASE, "weight"), {hc_mult}, 0);
|
||||
hc_head_scale = create_tensor(tn(LLM_TENSOR_HC_HEAD_SCALE, "weight"), {1}, 0);
|
||||
}
|
||||
|
||||
for (int i = 0; i < n_layer; ++i) {
|
||||
auto & layer = layers[i];
|
||||
@@ -1280,12 +1289,260 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_
|
||||
}
|
||||
}
|
||||
|
||||
// DeepSeek-V4.1 DSpark decoder (mtp.0..2), dual-mode by batch type like graph_dsv4:
|
||||
// * embd batch -> writes every stage's kv_norm(wkv(main_proj(target features))) into its window ring
|
||||
// * token batch -> [anchor, noise...] block through the stages, single-pass mHC, no per-head q norm
|
||||
llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const llm_graph_params & params) :
|
||||
llama_model_deepseek41::graph(params) {
|
||||
const int64_t n_embd_inp = hparams.n_embd_inp_enc();
|
||||
const int64_t n_embd_head = hparams.n_embd_head_k();
|
||||
const int64_t n_embd_head_rope = hparams.n_rot();
|
||||
const int64_t n_embd_head_nope = n_embd_head - n_embd_head_rope;
|
||||
|
||||
GGML_ASSERT(hparams.dsv4_hc_single_pass);
|
||||
|
||||
ggml_tensor * inp_pos = build_inp_pos();
|
||||
|
||||
llm_graph_input_attn_k_iswa * inp_attn = build_attn_inp_k_iswa();
|
||||
|
||||
// ratio-0 layers: plain rope at rope_theta, no YaRN (original_seq_len 0)
|
||||
const auto rope = [&](ggml_tensor * t) {
|
||||
t = ggml_rope_ext(ctx0, t, inp_pos, nullptr, n_embd_head_rope, rope_type, 0,
|
||||
freq_base, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
return ggml_rope_set_offset(t, n_embd_head_nope);
|
||||
};
|
||||
|
||||
ASSERT_EMBD_OR_TOKEN(ubatch);
|
||||
if (ubatch.embd) {
|
||||
auto inp = std::make_unique<llm_graph_input_embd>(n_embd_inp);
|
||||
|
||||
inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd_inp, n_tokens);
|
||||
ggml_set_input(inp->embd);
|
||||
|
||||
ggml_tensor * inp_target = inp->embd;
|
||||
cb(inp_target, "inp_target_features", -1);
|
||||
|
||||
res->add_input(std::move(inp));
|
||||
|
||||
ggml_tensor * inp_g = build_lora_mm(model.fc, inp_target, model.fc_s);
|
||||
inp_g = build_norm(inp_g, model.output_norm_enc, nullptr, LLM_NORM_RMS, -1);
|
||||
cb(inp_g, "main_x", -1);
|
||||
|
||||
for (int il = 0; il < n_layer; ++il) {
|
||||
const auto & layer = model.layers[il];
|
||||
|
||||
ggml_tensor * kv = build_lora_mm(layer.wkv, inp_g);
|
||||
kv = build_norm(kv, layer.attn_kv_norm, nullptr, LLM_NORM_RMS, il);
|
||||
kv = ggml_reshape_3d(ctx0, kv, n_embd_head, 1, n_tokens);
|
||||
kv = rope(kv);
|
||||
cb(kv, "kv_injected", il);
|
||||
|
||||
if (inp_attn->self_k_rot_swa) {
|
||||
kv = llama_mul_mat_hadamard(ctx0, kv, inp_attn->self_k_rot_swa);
|
||||
}
|
||||
ggml_build_forward_expand(gf, inp_attn->mctx->get_swa()->cpy_k(ctx0, kv, inp_attn->get_k_idxs_swa(), il));
|
||||
}
|
||||
|
||||
res->t_embd = inp_g;
|
||||
|
||||
ggml_build_forward_expand(gf, inp_g);
|
||||
return;
|
||||
}
|
||||
|
||||
auto * tok_embd = model.tok_embd;
|
||||
if (tok_embd == nullptr) {
|
||||
GGML_ASSERT(cparams.ctx_other != nullptr);
|
||||
const auto * model_other = llama_get_model(cparams.ctx_other);
|
||||
GGML_ASSERT(model_other->tok_embd != nullptr && "DSpark decoder requires the target model's token embeddings");
|
||||
tok_embd = model_other->tok_embd;
|
||||
}
|
||||
|
||||
auto inp = std::make_unique<llm_graph_input_embd>(n_embd);
|
||||
|
||||
inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
|
||||
ggml_set_input(inp->tokens);
|
||||
|
||||
ggml_tensor * inp_tokens = inp->tokens;
|
||||
|
||||
ggml_tensor * inpL = ggml_get_rows(ctx0, tok_embd, inp->tokens);
|
||||
cb(inpL, "inp_noise_embd", -1);
|
||||
|
||||
res->add_input(std::move(inp));
|
||||
|
||||
const int64_t hc = hparams.dsv4_hc_mult;
|
||||
const int64_t n_groups = hparams.dsv4_o_group_count;
|
||||
const int64_t n_heads_group = n_head / n_groups;
|
||||
const int64_t o_lora_rank = hparams.dsv4_o_lora_rank;
|
||||
const int64_t o_group_dim = n_heads_group*n_embd_head;
|
||||
const float kq_scale = 1.0f/sqrtf(float(n_embd_head));
|
||||
|
||||
GGML_ASSERT(n_embd_head == n_embd_head_v);
|
||||
GGML_ASSERT(n_head % n_groups == 0);
|
||||
|
||||
inpL = ggml_reshape_3d(ctx0, inpL, n_embd, 1, n_tokens);
|
||||
inpL = ggml_repeat_4d(ctx0, inpL, n_embd, hc, n_tokens, 1);
|
||||
cb(inpL, "hc_init", -1);
|
||||
|
||||
// window of a block = window of its first query, see n_swa
|
||||
{
|
||||
ggml_tensor * m = inp_attn->self_kq_mask_swa_cnv;
|
||||
const int64_t n_blocks = ubatch.n_seqs_unq;
|
||||
const int64_t bd = n_blocks > 0 ? n_tokens / n_blocks : 0;
|
||||
if (m && bd > 1 && n_tokens % n_blocks == 0 && m->ne[2] == 1 && m->ne[1] % bd == 0 && ggml_is_contiguous(m)) {
|
||||
ggml_tensor * m4 = ggml_reshape_4d(ctx0, m, m->ne[0], bd, m->ne[1]/bd, m->ne[3]);
|
||||
ggml_tensor * r0 = ggml_view_4d(ctx0, m4, m4->ne[0], 1, m4->ne[2], m4->ne[3], m4->nb[1], m4->nb[2], m4->nb[3], 0);
|
||||
m4 = ggml_repeat(ctx0, r0, m4);
|
||||
inp_attn->self_kq_mask_swa_cnv = ggml_reshape_4d(ctx0, m4, m->ne[0], m->ne[1], 1, m->ne[3]);
|
||||
cb(inp_attn->self_kq_mask_swa_cnv, "dspark_block_mask", -1);
|
||||
}
|
||||
}
|
||||
|
||||
ggml_tensor * pre_prev = nullptr;
|
||||
|
||||
for (int il = 0; il < n_layer; ++il) {
|
||||
const auto & layer = model.layers[il];
|
||||
|
||||
ggml_tensor * residual = inpL;
|
||||
ggml_tensor * post = nullptr;
|
||||
ggml_tensor * comb = nullptr;
|
||||
|
||||
ggml_tensor * attn_pre = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, false, il);
|
||||
|
||||
ggml_tensor * cur;
|
||||
if (pre_prev == nullptr) {
|
||||
// identity pre-mix of the first stage: copy 0
|
||||
cur = ggml_cont(ctx0, ggml_view_2d(ctx0, inpL, n_embd, n_tokens, inpL->nb[2], 0));
|
||||
} else {
|
||||
cur = build_hc_pre(inpL, pre_prev, il);
|
||||
}
|
||||
cb(cur, "hc_attn_pre", il);
|
||||
|
||||
cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
|
||||
cb(cur, "attn_norm", il);
|
||||
|
||||
// non-causal draft context: block queries see the main-KV ring plus every block key
|
||||
{
|
||||
ggml_tensor * qr = build_lora_mm(layer.wq_a, cur);
|
||||
qr = build_norm(qr, layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il);
|
||||
cb(qr, "qr_norm", il);
|
||||
|
||||
ggml_tensor * q = build_lora_mm(layer.wq_b, qr);
|
||||
q = ggml_reshape_3d(ctx0, q, n_embd_head, n_head, n_tokens);
|
||||
q = rope(q);
|
||||
cb(q, "q", il);
|
||||
|
||||
ggml_tensor * kv = build_lora_mm(layer.wkv, cur);
|
||||
kv = build_norm(kv, layer.attn_kv_norm, nullptr, LLM_NORM_RMS, il);
|
||||
kv = ggml_reshape_3d(ctx0, kv, n_embd_head, 1, n_tokens);
|
||||
kv = rope(kv);
|
||||
cb(kv, "kv", il);
|
||||
|
||||
ggml_tensor * out = build_attn(inp_attn,
|
||||
nullptr, nullptr, nullptr,
|
||||
q, kv, kv,
|
||||
nullptr, layer.attn_sinks, nullptr,
|
||||
kq_scale, il);
|
||||
cb(out, "attn_raw", il);
|
||||
|
||||
out = ggml_reshape_3d(ctx0, out, n_embd_head, n_head, n_tokens);
|
||||
out = ggml_rope_ext_back(ctx0, out, inp_pos, nullptr, n_embd_head_rope, rope_type, 0,
|
||||
freq_base, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
out = ggml_rope_set_offset(out, n_embd_head_nope);
|
||||
cb(out, "attn_derope", il);
|
||||
|
||||
out = ggml_reshape_3d(ctx0, out, o_group_dim, n_groups, n_tokens);
|
||||
out = ggml_permute(ctx0, out, 0, 2, 1, 3);
|
||||
ggml_tensor * oa = ggml_mul_mat(ctx0, layer.wo_a, out);
|
||||
cb(oa, "attn_wo_a", il);
|
||||
oa = ggml_permute(ctx0, oa, 0, 2, 1, 3);
|
||||
oa = ggml_cont_2d(ctx0, oa, o_lora_rank*n_groups, n_tokens);
|
||||
|
||||
cur = build_lora_mm(layer.wo_b, oa);
|
||||
cb(cur, "attn_out", il);
|
||||
}
|
||||
|
||||
inpL = build_hc_post(cur, residual, post, comb, il);
|
||||
cb(inpL, "hc_attn_post", il);
|
||||
|
||||
residual = inpL;
|
||||
ggml_tensor * ffn_pre = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, false, il);
|
||||
|
||||
cur = build_hc_pre(inpL, attn_pre, il);
|
||||
cb(cur, "hc_ffn_pre", il);
|
||||
|
||||
cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
|
||||
cb(cur, "ffn_norm", il);
|
||||
|
||||
ggml_tensor * moe_out = build_moe_ffn(cur,
|
||||
layer.ffn_gate_inp,
|
||||
layer.ffn_up_exps,
|
||||
layer.ffn_gate_exps,
|
||||
layer.ffn_down_exps,
|
||||
layer.ffn_exp_probs_b,
|
||||
n_expert, hparams.n_expert_used(),
|
||||
LLM_FFN_SILU, hparams.expert_weights_norm,
|
||||
hparams.expert_weights_scale,
|
||||
(llama_expert_gating_func_type) hparams.expert_gating_func,
|
||||
il);
|
||||
cb(moe_out, "ffn_moe_out", il);
|
||||
|
||||
ggml_tensor * ffn_shexp = build_ffn(cur,
|
||||
layer.ffn_up_shexp, nullptr, nullptr,
|
||||
layer.ffn_gate_shexp, nullptr, nullptr,
|
||||
layer.ffn_down_shexp, nullptr, nullptr,
|
||||
nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il);
|
||||
cb(ffn_shexp, "ffn_shexp", il);
|
||||
|
||||
cur = ggml_add(ctx0, moe_out, ffn_shexp);
|
||||
cb(cur, "ffn_out", il);
|
||||
|
||||
inpL = build_hc_post(cur, residual, post, comb, il);
|
||||
|
||||
ggml_tensor * l_out = ggml_reshape_2d(ctx0, inpL, n_embd*hc, n_tokens);
|
||||
cb(l_out, "l_out", il);
|
||||
ggml_build_forward_expand(gf, l_out);
|
||||
|
||||
pre_prev = ffn_pre;
|
||||
}
|
||||
|
||||
// no hc_head: collapse with the last FFN's pre-mix
|
||||
ggml_tensor * cur = build_hc_pre(inpL, pre_prev, -1);
|
||||
cb(cur, "hc_out", -1);
|
||||
|
||||
// confidence head takes the un-normed collapsed state
|
||||
res->t_embd = cur;
|
||||
|
||||
cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1);
|
||||
cb(cur, "result_norm", -1);
|
||||
|
||||
auto * output = model.output;
|
||||
auto * output_s = model.output_s;
|
||||
if (output == nullptr) {
|
||||
GGML_ASSERT(cparams.ctx_other != nullptr);
|
||||
const auto * model_other = llama_get_model(cparams.ctx_other);
|
||||
GGML_ASSERT(model_other->output != nullptr && "DSpark decoder requires the target model's output projection");
|
||||
output = model_other->output;
|
||||
output_s = model_other->output_s;
|
||||
}
|
||||
|
||||
cur = build_lora_mm(output, cur, output_s);
|
||||
cb(cur, "result_output", -1);
|
||||
res->t_logits = cur;
|
||||
|
||||
if (model.dspark_markov_w1) {
|
||||
build_dspark_markov_head(*this, model, inp_tokens);
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<llm_graph_context> llama_model_dflash::build_arch_graph(const llm_graph_params & params) const {
|
||||
switch (params.gtype) {
|
||||
case LLM_GRAPH_TYPE_ENCODER:
|
||||
return std::make_unique<graph<true>>(*this, params);
|
||||
case LLM_GRAPH_TYPE_DEFAULT:
|
||||
case LLM_GRAPH_TYPE_DECODER:
|
||||
if (hparams.dsv4_hc_mult > 0 && hparams.dsv4_hc_single_pass) {
|
||||
return std::make_unique<graph_dsv41>(*this, params);
|
||||
}
|
||||
if (hparams.dsv4_hc_mult > 0) {
|
||||
return std::make_unique<graph_dsv4>(*this, params);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user