diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp index 13f6a5acc..4d58b2fd2 100644 --- a/src/models/deepseek4.cpp +++ b/src/models/deepseek4.cpp @@ -342,24 +342,23 @@ ggml_tensor * llama_model_deepseek4::graph::build_hc_sinkhorn( return comb; } -ggml_tensor * llama_model_deepseek4::graph::build_hc_pre( +llama_model_deepseek4::graph::hc_pre +llama_model_deepseek4::graph::build_hc_pre( ggml_tensor * x, ggml_tensor * hc_fn, ggml_tensor * hc_scale, ggml_tensor * hc_base, - ggml_tensor ** post, - ggml_tensor ** comb, int il) const { - return build_hc_pre(x, build_hc_mixes(x, hc_fn, hc_scale, hc_base, post, comb, il), il); + hc_mixes mixes = build_hc_mixes(x, hc_fn, hc_scale, hc_base, il); + return { build_hc_pre(x, mixes.pre, il), mixes.post, mixes.comb }; } -ggml_tensor * llama_model_deepseek4::graph::build_hc_mixes( +llama_model_deepseek4::graph::hc_mixes +llama_model_deepseek4::graph::build_hc_mixes( ggml_tensor * x, ggml_tensor * hc_fn, ggml_tensor * hc_scale, ggml_tensor * hc_base, - ggml_tensor ** post, - ggml_tensor ** comb, int il) const { const int64_t hc = hparams.dsv4_hc_mult; const int64_t hc_dim = hc*n_embd; @@ -386,28 +385,29 @@ ggml_tensor * llama_model_deepseek4::graph::build_hc_mixes( pre = ggml_scale_bias(ctx0, pre, 1.0f, hparams.dsv4_hc_eps); cb(pre, "hc_pre", il); - *post = dsv4_view_2d(ctx0, mixes, hc, nt, hc); - *post = dsv4_hc_affine(ctx0, *post, scale_post, base_post); - *post = ggml_sigmoid(ctx0, *post); - *post = ggml_scale(ctx0, *post, 2.0f); - cb(*post, "hc_post", il); + ggml_tensor * post = dsv4_view_2d(ctx0, mixes, hc, nt, hc); + post = dsv4_hc_affine(ctx0, post, scale_post, base_post); + post = ggml_sigmoid(ctx0, post); + post = ggml_scale(ctx0, post, 2.0f); + cb(post, "hc_post", il); + ggml_tensor * comb = nullptr; if (cparams.fused_dsv4_hc_comb) { - *comb = ggml_dsv4_hc_comb(ctx0, mixes, hc_scale, hc_base, hparams.dsv4_hc_eps, + comb = ggml_dsv4_hc_comb(ctx0, mixes, hc_scale, hc_base, hparams.dsv4_hc_eps, (int32_t) hparams.dsv4_hc_sinkhorn_iters); - res->add_fused_node({LLM_FUSED_OP_DSV4_HC_COMB, *comb, il}); + res->add_fused_node({LLM_FUSED_OP_DSV4_HC_COMB, comb, il}); } else { ggml_tensor * scale_comb = dsv4_view_1d(ctx0, hc_scale, 1, 2); ggml_tensor * base_comb = dsv4_view_1d(ctx0, hc_base, hc*hc, 2*hc); - *comb = dsv4_view_2d(ctx0, mixes, hc*hc, nt, 2*hc); - *comb = dsv4_hc_affine(ctx0, *comb, scale_comb, base_comb); - *comb = ggml_reshape_3d(ctx0, *comb, hc, hc, nt); - *comb = build_hc_sinkhorn(*comb, il); + comb = dsv4_view_2d(ctx0, mixes, hc*hc, nt, 2*hc); + comb = dsv4_hc_affine(ctx0, comb, scale_comb, base_comb); + comb = ggml_reshape_3d(ctx0, comb, hc, hc, nt); + comb = build_hc_sinkhorn(comb, il); } - cb(*comb, "hc_comb", il); + cb(comb, "hc_comb", il); - return pre; + return { pre, post, comb }; } ggml_tensor * llama_model_deepseek4::graph::build_hc_post( @@ -1252,14 +1252,12 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p } ggml_tensor * residual = inpL; - ggml_tensor * post = nullptr; - ggml_tensor * comb = nullptr; - - cur = build_hc_pre(inpL, + const hc_pre attn_hc = build_hc_pre(inpL, model.layers[il].hc_attn_fn, model.layers[il].hc_attn_scale, model.layers[il].hc_attn_base, - &post, &comb, il); + il); + cur = attn_hc.cur; cb(cur, "hc_attn_pre", il); cur = build_norm(cur, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); @@ -1267,20 +1265,21 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p cur = build_attention(model, inp_dsv4, cur, inp_pos, il); - inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_hc_post(cur, residual, attn_hc.post, attn_hc.comb, il); cb(inpL, "hc_attn_post", il); residual = inpL; - cur = build_hc_pre(inpL, + const hc_pre ffn_hc = build_hc_pre(inpL, model.layers[il].hc_ffn_fn, model.layers[il].hc_ffn_scale, model.layers[il].hc_ffn_base, - &post, &comb, il); + il); + cur = ffn_hc.cur; cb(cur, "hc_ffn_pre", il); ggml_build_forward_expand(gf, residual); - ggml_build_forward_expand(gf, post); - ggml_build_forward_expand(gf, comb); + ggml_build_forward_expand(gf, ffn_hc.post); + ggml_build_forward_expand(gf, ffn_hc.comb); cur = build_norm(cur, model.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "ffn_norm", il); @@ -1330,7 +1329,7 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p cur = ggml_add(ctx0, moe_out, ffn_shexp); cb(cur, "ffn_out", il); - inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_hc_post(cur, residual, ffn_hc.post, ffn_hc.comb, il); inpL = build_cvec(inpL, il); cb(inpL, "l_last", il); } @@ -1430,14 +1429,12 @@ llama_model_deepseek4::graph_mtp::graph_mtp(const llama_model & model, const llm cb(inpL, "mtp_eh_proj", il); ggml_tensor * residual = inpL; - ggml_tensor * post = nullptr; - ggml_tensor * comb = nullptr; - - ggml_tensor * cur = build_hc_pre(inpL, + hc_pre attn_hc = build_hc_pre(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, - &post, &comb, il); + il); + ggml_tensor * cur = attn_hc.cur; cb(cur, "mtp_hc_attn_pre", il); cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); @@ -1445,15 +1442,16 @@ llama_model_deepseek4::graph_mtp::graph_mtp(const llama_model & model, const llm cur = build_attention(model, inp_attn, cur, inp_pos, il); - inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_hc_post(cur, residual, attn_hc.post, attn_hc.comb, il); cb(inpL, "mtp_hc_attn_post", il); residual = inpL; - cur = build_hc_pre(inpL, + hc_pre ffn_hc = build_hc_pre(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, - &post, &comb, il); + il); + cur = ffn_hc.cur; cb(cur, "mtp_hc_ffn_pre", il); cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); @@ -1483,7 +1481,7 @@ llama_model_deepseek4::graph_mtp::graph_mtp(const llama_model & model, const llm cur = ggml_add(ctx0, moe_out, ffn_shexp); cb(cur, "mtp_ffn_out", il); - inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_hc_post(cur, residual, ffn_hc.post, ffn_hc.comb, il); inpL = build_cvec(inpL, il); cb(inpL, "mtp_l_out", il); diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp index 6a9003d42..97b2cbabe 100644 --- a/src/models/deepseek41.cpp +++ b/src/models/deepseek41.cpp @@ -847,10 +847,7 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g } ggml_tensor * residual = inpL; - ggml_tensor * post = nullptr; - ggml_tensor * comb = nullptr; - - ggml_tensor * attn_pre = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, il); + const hc_mixes attn_mix = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, il); // the first layer's attention collapses with the identity pre-mix: copy 0 cur = pre_prev ? build_hc_pre(inpL, pre_prev, il) : ggml_cont(ctx0, ggml_view_2d(ctx0, inpL, n_embd, n_tokens, inpL->nb[2], 0)); @@ -861,18 +858,19 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g cur = build_attention41(model, inp_dsv4, cur, inp_pos, cand_pin, il); - inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_hc_post(cur, residual, attn_mix.post, attn_mix.comb, il); cb(inpL, "hc_attn_post", il); residual = inpL; - pre_prev = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, il); + const hc_mixes ffn_mix = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, il); + pre_prev = ffn_mix.pre; - cur = build_hc_pre(inpL, attn_pre, il); + cur = build_hc_pre(inpL, attn_mix.pre, il); cb(cur, "hc_ffn_pre", il); ggml_build_forward_expand(gf, residual); - ggml_build_forward_expand(gf, post); - ggml_build_forward_expand(gf, comb); + ggml_build_forward_expand(gf, ffn_mix.post); + ggml_build_forward_expand(gf, ffn_mix.comb); cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "ffn_norm", il); @@ -912,7 +910,7 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g cur = ggml_add(ctx0, moe_out, ffn_shexp); cb(cur, "ffn_out", il); - inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_hc_post(cur, residual, ffn_mix.post, ffn_mix.comb, il); inpL = build_cvec(inpL, il); cb(inpL, "l_out", il); } diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp index 6bbb534f0..e3c2f1f29 100644 --- a/src/models/dflash.cpp +++ b/src/models/dflash.cpp @@ -1204,14 +1204,12 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_ const auto & layer = model.layers[il]; ggml_tensor * residual = inpL; - ggml_tensor * post = nullptr; - ggml_tensor * comb = nullptr; - - ggml_tensor * cur = build_hc_pre(inpL, + hc_pre attn_hc = build_hc_pre(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, - &post, &comb, il); + il); + ggml_tensor * cur = attn_hc.cur; cb(cur, "hc_attn_pre", il); cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); @@ -1219,15 +1217,16 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_ cur = build_attention(model, inp_attn, cur, inp_pos, il); - inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_hc_post(cur, residual, attn_hc.post, attn_hc.comb, il); cb(inpL, "hc_attn_post", il); residual = inpL; - cur = build_hc_pre(inpL, + hc_pre ffn_hc = build_hc_pre(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, - &post, &comb, il); + il); + cur = ffn_hc.cur; cb(cur, "hc_ffn_pre", il); cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); @@ -1256,7 +1255,7 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_ cur = ggml_add(ctx0, moe_out, ffn_shexp); cb(cur, "ffn_out", il); - inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_hc_post(cur, residual, ffn_hc.post, ffn_hc.comb, il); cb(inpL, "l_out", il); } @@ -1403,10 +1402,7 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll const auto & layer = model.layers[il]; ggml_tensor * residual = inpL; - ggml_tensor * post = nullptr; - ggml_tensor * comb = nullptr; - - ggml_tensor * attn_pre = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, il); + const hc_mixes attn_mix = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, il); ggml_tensor * cur; if (pre_prev == nullptr) { @@ -1461,13 +1457,14 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll cb(cur, "attn_out", il); } - inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_hc_post(cur, residual, attn_mix.post, attn_mix.comb, il); cb(inpL, "hc_attn_post", il); residual = inpL; - ggml_tensor * ffn_pre = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, il); + const hc_mixes ffn_mix = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, il); + pre_prev = ffn_mix.pre; - cur = build_hc_pre(inpL, attn_pre, il); + cur = build_hc_pre(inpL, attn_mix.pre, il); cb(cur, "hc_ffn_pre", il); cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); @@ -1496,13 +1493,11 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll cur = ggml_add(ctx0, moe_out, ffn_shexp); cb(cur, "ffn_out", il); - inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_hc_post(cur, residual, ffn_mix.post, ffn_mix.comb, il); ggml_tensor * l_out = ggml_reshape_2d(ctx0, inpL, n_embd*hc, n_tokens); cb(l_out, "l_out", il); ggml_build_forward_expand(gf, l_out); - - pre_prev = ffn_pre; } // no hc_head: collapse with the last FFN's pre-mix diff --git a/src/models/glm5-next.cpp b/src/models/glm5-next.cpp index 395560990..d1e9b503d 100644 --- a/src/models/glm5-next.cpp +++ b/src/models/glm5-next.cpp @@ -442,13 +442,12 @@ ggml_tensor * llama_model_glm5_next::graph::build_hc_sinkhorn( return comb; } -ggml_tensor * llama_model_glm5_next::graph::build_hc_pre( +llama_model_glm5_next::graph::hc_pre +llama_model_glm5_next::graph::build_hc_pre( ggml_tensor * x, ggml_tensor * hc_fn, ggml_tensor * hc_scale, ggml_tensor * hc_base, - ggml_tensor ** post, - ggml_tensor ** comb, int il) const { const int64_t hc = hparams.dsv4_hc_mult; const int64_t hc_dim = hc*n_embd; @@ -475,29 +474,29 @@ ggml_tensor * llama_model_glm5_next::graph::build_hc_pre( pre = ggml_scale_bias(ctx0, pre, 1.0f, hparams.dsv4_hc_eps); cb(pre, "hc_pre", il); - *post = dsv4_view_2d(ctx0, mixes, hc, nt, hc); - *post = dsv4_hc_affine(ctx0, *post, scale_post, base_post); - *post = ggml_sigmoid(ctx0, *post); - *post = ggml_scale(ctx0, *post, 2.0f); - cb(*post, "hc_post", il); + ggml_tensor * post = dsv4_view_2d(ctx0, mixes, hc, nt, hc); + post = dsv4_hc_affine(ctx0, post, scale_post, base_post); + post = ggml_sigmoid(ctx0, post); + post = ggml_scale(ctx0, post, 2.0f); + cb(post, "hc_post", il); + ggml_tensor * comb = nullptr; if (cparams.fused_dsv4_hc_comb) { - *comb = ggml_dsv4_hc_comb(ctx0, mixes, hc_scale, hc_base, hparams.dsv4_hc_eps, + comb = ggml_dsv4_hc_comb(ctx0, mixes, hc_scale, hc_base, hparams.dsv4_hc_eps, (int32_t) hparams.dsv4_hc_sinkhorn_iters); - res->add_fused_node({LLM_FUSED_OP_DSV4_HC_COMB, *comb, il}); + res->add_fused_node({LLM_FUSED_OP_DSV4_HC_COMB, comb, il}); } else { ggml_tensor * scale_comb = dsv4_view_1d(ctx0, hc_scale, 1, 2); ggml_tensor * base_comb = dsv4_view_1d(ctx0, hc_base, hc*hc, 2*hc); - *comb = dsv4_view_2d(ctx0, mixes, hc*hc, nt, 2*hc); - *comb = dsv4_hc_affine(ctx0, *comb, scale_comb, base_comb); - *comb = ggml_reshape_3d(ctx0, *comb, hc, hc, nt); - *comb = build_hc_sinkhorn(*comb, il); + comb = dsv4_view_2d(ctx0, mixes, hc*hc, nt, 2*hc); + comb = dsv4_hc_affine(ctx0, comb, scale_comb, base_comb); + comb = ggml_reshape_3d(ctx0, comb, hc, hc, nt); + comb = build_hc_sinkhorn(comb, il); } - cb(*comb, "hc_comb", il); + cb(comb, "hc_comb", il); - ggml_tensor * result = build_hc_pre(x, pre, il); - return result; + return { build_hc_pre(x, pre, il), post, comb }; } ggml_tensor * llama_model_glm5_next::graph::build_hc_post( @@ -609,8 +608,9 @@ llama_model_glm5_next::graph_mtp::graph_mtp(const llama_model & model, const llm cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "mtp_attn_norm", il); - ggml_tensor * prev_sel = nullptr; - cur = build_dsa_layer(cur, layer, mctx_hyb, inp_attn, inp_kpool, &prev_sel, il); + // the NextN block has a full indexer, nothing to carry in + dsa_out dsa = build_dsa_layer(cur, layer, mctx_hyb, inp_attn, inp_kpool, nullptr, il); + cur = dsa.cur; cb(cur, "mtp_attn_out", il); // narrow to the output tokens before the position-wise FFN; unmasked nextn embeddings need all rows @@ -711,10 +711,8 @@ llama_model_glm5_next::graph::graph(const llama_model & model, const llm_graph_p const auto & layer = model.layers[il]; ggml_tensor * residual = inpL; - ggml_tensor * post = nullptr; - ggml_tensor * comb = nullptr; - - cur = build_hc_pre(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, il); + hc_pre attn_hc = build_hc_pre(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, il); + cur = attn_hc.cur; cb(cur, "hc_attn_pre", il); cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); @@ -725,14 +723,17 @@ llama_model_glm5_next::graph::graph(const llama_model & model, const llm_graph_p cur = build_kda_layer(cur, layer, inp_rs, d_conv, head_dim, n_head_kda, d_inner, n_seq_tokens, n_seqs, il); } else { - cur = build_dsa_layer(cur, layer, mctx_hyb, inp_attn, inp_kpool, &prev_sel, il); + dsa_out dsa = build_dsa_layer(cur, layer, mctx_hyb, inp_attn, inp_kpool, prev_sel, il); + cur = dsa.cur; + prev_sel = dsa.sel; } - inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_hc_post(cur, residual, attn_hc.post, attn_hc.comb, il); cb(inpL, "hc_attn_post", il); residual = inpL; - cur = build_hc_pre(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, il); + hc_pre ffn_hc = build_hc_pre(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, il); + cur = ffn_hc.cur; cb(cur, "hc_ffn_pre", il); cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); @@ -770,7 +771,7 @@ llama_model_glm5_next::graph::graph(const llama_model & model, const llm_graph_p cb(cur, "ffn_out", il); } - inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_hc_post(cur, residual, ffn_hc.post, ffn_hc.comb, il); inpL = build_cvec(inpL, il); cb(inpL, "l_out", il); } @@ -1039,10 +1040,11 @@ ggml_tensor * llama_model_glm5_next::graph::build_kpool_select( // Nope MLA layer with sparse attention over the indexer selection -ggml_tensor * llama_model_glm5_next::graph::build_dsa_layer( +llama_model_glm5_next::graph::dsa_out +llama_model_glm5_next::graph::build_dsa_layer( ggml_tensor * cur, const llama_layer & layer, const llama_memory_hybrid_idx_context * mctx_hyb, llm_graph_input_attn_k * inp_attn, - llm_graph_input_kpool * inp_kpool, ggml_tensor ** prev_sel, int il) { + llm_graph_input_kpool * inp_kpool, ggml_tensor * prev_sel, int il) { const auto * mctx_mla = mctx_hyb->get_attn(); @@ -1073,13 +1075,11 @@ ggml_tensor * llama_model_glm5_next::graph::build_dsa_layer( ggml_tensor * kq_mask = inp_attn->get_kq_mask(); - ggml_tensor * sel = nullptr; + ggml_tensor * sel = prev_sel; if (il >= (int) hparams.n_layer() || hparams.is_indexer_full(il)) { // the NextN block always has a full indexer sel = build_kpool_select(cur, qr, kq_mask, layer, mctx_hyb, inp_kpool, il); - *prev_sel = sel; } else { - GGML_ASSERT(*prev_sel != nullptr && "shared indexer layer must follow a full indexer layer"); - sel = *prev_sel; + GGML_ASSERT(sel != nullptr && "shared indexer layer must follow a full indexer layer"); } ggml_build_forward_expand(gf, q_absorbed); @@ -1099,5 +1099,5 @@ ggml_tensor * llama_model_glm5_next::graph::build_dsa_layer( out = ggml_mul_mat(ctx0, layer.wo, out); cb(out, "attn_out", il); - return out; + return { out, sel }; } diff --git a/src/models/hy-v4.cpp b/src/models/hy-v4.cpp index 25e40abf7..3d456fe85 100644 --- a/src/models/hy-v4.cpp +++ b/src/models/hy-v4.cpp @@ -172,12 +172,12 @@ static ggml_tensor * hy_v4_hc_reduce(ggml_context * ctx0, ggml_tensor * x, ggml_ return ggml_cast(ctx0, result, out_type); } -ggml_tensor * llama_model_hy_v4::graph::build_hc_pre( +llama_model_hy_v4::graph::hc_pre +llama_model_hy_v4::graph::build_hc_pre( ggml_tensor * x, ggml_tensor * hc_fn, ggml_tensor * hc_scale, ggml_tensor * hc_base, - ggml_tensor ** post, int il) const { const int64_t hc = hparams.dsv4_hc_mult; const int64_t nt = x->ne[2]; @@ -202,16 +202,15 @@ ggml_tensor * llama_model_hy_v4::graph::build_hc_pre( cb(pre, "hc_pre", il); // post = magnitude*sigmoid(mixes[hc:2hc]*scale_post + base_post) + eps - ggml_tensor * po = hy_v4_view_2d(ctx0, mixes, hc, nt, hc); - po = ggml_mul(ctx0, po, scale_post); - po = ggml_add(ctx0, po, base_post); - po = ggml_sigmoid(ctx0, po); - po = ggml_scale(ctx0, po, hparams.hc_magnitude); - po = ggml_scale_bias(ctx0, po, 1.0f, hparams.dsv4_hc_eps); - *post = po; - cb(po, "hc_post_gate", il); + ggml_tensor * post = hy_v4_view_2d(ctx0, mixes, hc, nt, hc); + post = ggml_mul(ctx0, post, scale_post); + post = ggml_add(ctx0, post, base_post); + post = ggml_sigmoid(ctx0, post); + post = ggml_scale(ctx0, post, hparams.hc_magnitude); + post = ggml_scale_bias(ctx0, post, 1.0f, hparams.dsv4_hc_eps); + cb(post, "hc_post_gate", il); - return hy_v4_hc_reduce(ctx0, x, pre, hc, n_embd, nt, x->type); + return { hy_v4_hc_reduce(ctx0, x, pre, hc, n_embd, nt, x->type), post }; } ggml_tensor * llama_model_hy_v4::graph::build_hc_post( @@ -418,12 +417,13 @@ ggml_tensor * llama_model_hy_v4::graph::build_indexer_top_k( return ggml_cont(ctx0, ggml_top_k(ctx0, score, n_top_k)); } -ggml_tensor * llama_model_hy_v4::graph::build_attention_dsa( +llama_model_hy_v4::graph::attn_dsa +llama_model_hy_v4::graph::build_attention_dsa( const llama_model & model, llm_graph_input_attn_k_dsa * inp_attn_dsa, ggml_tensor * cur, ggml_tensor * inp_pos, - ggml_tensor ** last_top_k, + ggml_tensor * last_top_k, float kq_scale, int il) const { const auto & layer = model.layers[il]; @@ -436,11 +436,12 @@ ggml_tensor * llama_model_hy_v4::graph::build_attention_dsa( ggml_tensor * qr = ggml_mul_mat(ctx0, layer.wq_a, cur); qr = build_norm(qr, layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il); + ggml_tensor * top_k = last_top_k; if (hparams.is_indexer_full(il)) { - *last_top_k = build_indexer_top_k(model, inp_attn_dsa, cur, qr, inp_pos, il); - cb(*last_top_k, "top_k", il); + top_k = build_indexer_top_k(model, inp_attn_dsa, cur, qr, inp_pos, il); + cb(top_k, "top_k", il); } - GGML_ASSERT(*last_top_k != nullptr); + GGML_ASSERT(top_k != nullptr); ggml_tensor * q = ggml_mul_mat(ctx0, layer.wq_b, qr); @@ -480,7 +481,7 @@ ggml_tensor * llama_model_hy_v4::graph::build_attention_dsa( ggml_tensor * attn = build_attn(inp_attn_dsa, nullptr, nullptr, nullptr, - Qcur, Kcur, Vcur, nullptr, layer.attn_sinks, layer.wv_b, *last_top_k, kq_scale, il); + Qcur, Kcur, Vcur, nullptr, layer.attn_sinks, layer.wv_b, top_k, kq_scale, il); cb(attn, "attn_kqv", il); ggml_tensor * gate = ggml_mul_mat(ctx0, layer.wqkv_gate, cur); @@ -491,7 +492,7 @@ ggml_tensor * llama_model_hy_v4::graph::build_attention_dsa( ggml_tensor * out = build_lora_mm(layer.wo, attn); cb(out, "attn_out", il); - return out; + return { out, top_k }; } llama_model_hy_v4::graph::graph(const llama_model & model, const llm_graph_params & params) : @@ -520,23 +521,27 @@ llama_model_hy_v4::graph::graph(const llama_model & model, const llm_graph_param for (int il = 0; il < n_layer; ++il) { ggml_tensor * residual = inpL; - ggml_tensor * post = nullptr; - - cur = build_hc_pre(inpL, model.layers[il].hc_attn_fn, model.layers[il].hc_attn_scale, - model.layers[il].hc_attn_base, &post, il); + hc_pre attn_hc = build_hc_pre(inpL, model.layers[il].hc_attn_fn, model.layers[il].hc_attn_scale, + model.layers[il].hc_attn_base, il); + cur = attn_hc.cur; cur = build_norm(cur, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); - cur = is_dsa - ? build_attention_dsa(model, inp_attn_dsa, cur, inp_pos, &last_top_k, kq_scale, il) - : build_attention(model, inp_attn, cur, inp_pos, kq_scale, il); + if (is_dsa) { + attn_dsa attn = build_attention_dsa(model, inp_attn_dsa, cur, inp_pos, last_top_k, kq_scale, il); + cur = attn.cur; + last_top_k = attn.top_k; + } else { + cur = build_attention(model, inp_attn, cur, inp_pos, kq_scale, il); + } - inpL = build_hc_post(cur, residual, post, il); + inpL = build_hc_post(cur, residual, attn_hc.post, il); cb(inpL, "hc_attn_out", il); residual = inpL; - cur = build_hc_pre(inpL, model.layers[il].hc_ffn_fn, model.layers[il].hc_ffn_scale, - model.layers[il].hc_ffn_base, &post, il); + hc_pre ffn_hc = build_hc_pre(inpL, model.layers[il].hc_ffn_fn, model.layers[il].hc_ffn_scale, + model.layers[il].hc_ffn_base, il); + cur = ffn_hc.cur; cur = build_norm(cur, model.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "ffn_norm", il); @@ -575,7 +580,7 @@ llama_model_hy_v4::graph::graph(const llama_model & model, const llm_graph_param cb(cur, "ffn_out", il); } - inpL = build_hc_post(cur, residual, post, il); + inpL = build_hc_post(cur, residual, ffn_hc.post, il); cb(inpL, "l_out", il); } diff --git a/src/models/models.h b/src/models/models.h index 37dbacdb9..5d9b6077a 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -1358,24 +1358,33 @@ struct llama_model_deepseek4 : public llama_model_base { // mean over the hyper-connection streams: [n_embd, hc, n_tokens] -> [n_embd, n_tokens] ggml_tensor * build_hc_mean(ggml_tensor * x) const; - // returns the collapsed input and fills the post / comb weights - ggml_tensor * build_hc_pre( + // all outputs of the hc mix builders: + // build_hc_pre - cur is the collapsed input, post / comb feed build_hc_post + // build_hc_mixes - pre is the pre weights, post / comb feed build_hc_post + struct hc_pre { + ggml_tensor * cur; + ggml_tensor * post; + ggml_tensor * comb; + }; + + struct hc_mixes { + ggml_tensor * pre; + ggml_tensor * post; + ggml_tensor * comb; + }; + + hc_pre build_hc_pre( ggml_tensor * x, ggml_tensor * hc_fn, ggml_tensor * hc_scale, ggml_tensor * hc_base, - ggml_tensor ** post, - ggml_tensor ** comb, int il) const; - // returns the pre weights and fills post / comb - ggml_tensor * build_hc_mixes( + hc_mixes build_hc_mixes( ggml_tensor * x, ggml_tensor * hc_fn, ggml_tensor * hc_scale, ggml_tensor * hc_base, - ggml_tensor ** post, - ggml_tensor ** comb, int il) const; ggml_tensor * build_hc_post( @@ -2263,12 +2272,16 @@ struct llama_model_hy_v4 : public llama_model_base { // iHC (independent Hyper-Connections): pre reduces the hc streams to one and returns the // per-stream post gates, post writes the sublayer output back into the streams, head // collapses the streams before the final norm. - ggml_tensor * build_hc_pre( + struct hc_pre { + ggml_tensor * cur; // the collapsed stream + ggml_tensor * post; // per-stream post gates, feeds build_hc_post + }; + + hc_pre build_hc_pre( ggml_tensor * x, ggml_tensor * hc_fn, ggml_tensor * hc_scale, ggml_tensor * hc_base, - ggml_tensor ** post, int il) const; ggml_tensor * build_hc_post( @@ -2301,12 +2314,18 @@ struct llama_model_hy_v4 : public llama_model_base { ggml_tensor * inp_pos, int il) const; - ggml_tensor * build_attention_dsa( + // the attention output and the top-k to carry to the following "shared" layers + struct attn_dsa { + ggml_tensor * cur; + ggml_tensor * top_k; + }; + + attn_dsa build_attention_dsa( const llama_model & model, llm_graph_input_attn_k_dsa * inp_attn_dsa, ggml_tensor * cur, ggml_tensor * inp_pos, - ggml_tensor ** last_top_k, + ggml_tensor * last_top_k, float kq_scale, int il) const; }; @@ -2730,14 +2749,18 @@ struct llama_model_qwen4exp : public llama_model_base { llm_build_delta_net_base(params), model(model) {} // HC replaces every layer norm: residual is [n_embd, hc, n_tokens] - ggml_tensor * build_hc_mix( + struct hc_mix { + ggml_tensor * cur; // the mixed [n_embd, T] stream + ggml_tensor * inject; // [hc, T] scatter weights, null when the inject weights are null + }; + + hc_mix build_hc_mix( ggml_tensor * x, ggml_tensor * w_norm, ggml_tensor * w_down, ggml_tensor * w_up, ggml_tensor * w_inject, ggml_tensor * w_down_inject, // [w_down | w_inject] in one weight, can be null - ggml_tensor ** inject, int il); ggml_tensor * build_hc_combine( @@ -2969,14 +2992,18 @@ struct llama_model_glm5_next : public llama_model_base { // mean over the hyper-connection streams: [n_embd, hc, n_tokens] -> [n_embd, n_tokens] ggml_tensor * build_hc_mean(ggml_tensor * x) const; - // returns the collapsed input and fills the post / comb weights - ggml_tensor * build_hc_pre( + // all outputs of the hc mix builder: cur is the collapsed input, post / comb feed build_hc_post + struct hc_pre { + ggml_tensor * cur; + ggml_tensor * post; + ggml_tensor * comb; + }; + + hc_pre build_hc_pre( ggml_tensor * x, ggml_tensor * hc_fn, ggml_tensor * hc_scale, ggml_tensor * hc_base, - ggml_tensor ** post, - ggml_tensor ** comb, int il) const; ggml_tensor * build_hc_post( @@ -3002,9 +3029,15 @@ struct llama_model_glm5_next : public llama_model_base { ggml_tensor * build_kpool_select(ggml_tensor * cur, ggml_tensor * qr, ggml_tensor * kq_mask, const llama_layer & layer, const llama_memory_hybrid_idx_context * mctx_hyb, llm_graph_input_kpool * inp_kpool, int il); - ggml_tensor * build_dsa_layer(ggml_tensor * cur, const llama_layer & layer, - const llama_memory_hybrid_idx_context * mctx_hyb, llm_graph_input_attn_k * inp_attn, - llm_graph_input_kpool * inp_kpool, ggml_tensor ** prev_sel, int il); + // attention output (cur) and the selection to carry to the next shared layer (sel) + struct dsa_out { + ggml_tensor * cur; + ggml_tensor * sel; + }; + + dsa_out build_dsa_layer(ggml_tensor * cur, const llama_layer & layer, + const llama_memory_hybrid_idx_context * mctx_hyb, llm_graph_input_attn_k * inp_attn, + llm_graph_input_kpool * inp_kpool, ggml_tensor * prev_sel, int il); }; diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index 1f3b1cdef..f5a81f8f6 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -343,15 +343,15 @@ static ggml_tensor * hc_stream_mean(ggml_context * ctx, ggml_tensor * x) { } // Hyper-connections keep hc parallel residual streams [n_embd, hc, T] in place of layer norms. -// Returns the mixed [n_embd, T] stream; `inject` gets the [hc, T] scatter weights. -ggml_tensor * llama_model_qwen4exp::graph::build_hc_mix( +// build_hc_mix returns the mixed [n_embd, T] stream and the [hc, T] scatter weights. +llama_model_qwen4exp::graph::hc_mix +llama_model_qwen4exp::graph::build_hc_mix( ggml_tensor * x, ggml_tensor * w_norm, ggml_tensor * w_down, ggml_tensor * w_up, ggml_tensor * w_inject, ggml_tensor * w_down_inject, - ggml_tensor ** inject, int il) { const int64_t hc = hparams.dsv4_hc_mult; const int64_t hc_dim = hc * n_embd; @@ -364,15 +364,16 @@ ggml_tensor * llama_model_qwen4exp::graph::build_hc_mix( cb(xn, "hc_norm", il); ggml_tensor * lo = nullptr; - if (inject && w_down_inject) { + ggml_tensor * inject = nullptr; + if (w_down_inject) { // one matmul for both: the hc_down rows, then the hc_inject rows const int64_t n_lo = w_down_inject->ne[1] - hc; ggml_tensor * lo_inject = build_lora_mm(w_down_inject, xn); // scale the combined rows before the split; both lo and inject use 1/hc lo_inject = ggml_scale(ctx0, lo_inject, 1.0f / (float) hc); - lo = ggml_view_2d(ctx0, lo_inject, n_lo, nt, lo_inject->nb[1], 0); - *inject = ggml_view_2d(ctx0, lo_inject, hc, nt, lo_inject->nb[1], ggml_row_size(lo_inject->type, n_lo)); - cb(*inject, "hc_inject", il); + lo = ggml_view_2d(ctx0, lo_inject, n_lo, nt, lo_inject->nb[1], 0); + inject = ggml_view_2d(ctx0, lo_inject, hc, nt, lo_inject->nb[1], ggml_row_size(lo_inject->type, n_lo)); + cb(inject, "hc_inject", il); } else { lo = ggml_scale(ctx0, build_lora_mm(w_down, xn), 1.0f / (float) hc); } @@ -405,12 +406,12 @@ ggml_tensor * llama_model_qwen4exp::graph::build_hc_mix( } cb(mixed, "hc_mixed", il); - if (inject && !w_down_inject) { - *inject = ggml_scale(ctx0, build_lora_mm(w_inject, xn), 1.0f / (float) hc); - cb(*inject, "hc_inject", il); + if (!w_down_inject && w_inject) { + inject = ggml_scale(ctx0, build_lora_mm(w_inject, xn), 1.0f / (float) hc); + cb(inject, "hc_inject", il); } - return mixed; + return { mixed, inject }; } ggml_tensor * llama_model_qwen4exp::graph::build_hc_combine( @@ -504,14 +505,14 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa res_hc = build_ple(inp->get_recr(), ple_emb, res_hc, il); } - ggml_tensor * inject = nullptr; - ggml_tensor * cur = build_hc_mix(res_hc, + hc_mix attn_mix = build_hc_mix(res_hc, model.layers[il].hc_attn_norm, model.layers[il].hc_attn_down, model.layers[il].hc_attn_up, model.layers[il].hc_attn_inject, model.layers[il].hc_attn_down_inject, - &inject, il); + il); + ggml_tensor * cur = attn_mix.cur; ggml_build_forward_expand(gf, cur); @@ -523,28 +524,29 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) { // everything below is per token, so drop the rows that produce no output - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inject = ggml_get_rows(ctx0, inject, inp_out_ids); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + attn_mix.inject = ggml_get_rows(ctx0, attn_mix.inject, inp_out_ids); res_hc = ggml_reshape_2d(ctx0, res_hc, n_embd*hc, res_hc->ne[2]); res_hc = ggml_get_rows(ctx0, res_hc, inp_out_ids); res_hc = ggml_reshape_3d(ctx0, res_hc, n_embd, hc, res_hc->ne[1]); } - res_hc = build_hc_combine(res_hc, cur, inject, il); + res_hc = build_hc_combine(res_hc, cur, attn_mix.inject, il); - cur = build_hc_mix(res_hc, + hc_mix ffn_mix = build_hc_mix(res_hc, model.layers[il].hc_ffn_norm, model.layers[il].hc_ffn_down, model.layers[il].hc_ffn_up, model.layers[il].hc_ffn_inject, model.layers[il].hc_ffn_down_inject, - &inject, il); + il); + cur = ffn_mix.cur; cur = build_layer_ffn(cur, il); cb(cur, "ffn_out", il); - res_hc = build_hc_combine(res_hc, cur, inject, il); + res_hc = build_hc_combine(res_hc, cur, ffn_mix.inject, il); // "l_last" is the layer output name that build_cvec and imatrix look for cb(res_hc, "l_last", il); @@ -562,9 +564,10 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa } // the final mixer is the output norm: there is no separate one - ggml_tensor * cur = build_hc_mix(res_hc, + hc_mix head_mix = build_hc_mix(res_hc, model.hc_head_norm, model.hc_head_down, model.hc_head_up, - nullptr, nullptr, nullptr, -1); + nullptr, nullptr, -1); + ggml_tensor * cur = head_mix.cur; cb(cur, "result_norm", -1); res->t_embd = cur; @@ -641,14 +644,15 @@ llama_model_qwen4exp::graph_mtp::graph_mtp(const llama_model & model, const llm_ ggml_tensor * res_hc = build_lora_mm(layer.nextn.eh_proj, ggml_concat(ctx0, e_norm, h_norm, 0)); // [n_embd, hc, n_tokens] cb(res_hc, "mtp_eh_proj", il); - ggml_tensor * inject = nullptr; - ggml_tensor * cur = build_hc_mix(res_hc, layer.hc_attn_norm, layer.hc_attn_down, layer.hc_attn_up, layer.hc_attn_inject, layer.hc_attn_down_inject, &inject, il); + hc_mix attn_mix = build_hc_mix(res_hc, layer.hc_attn_norm, layer.hc_attn_down, layer.hc_attn_up, layer.hc_attn_inject, layer.hc_attn_down_inject, il); + ggml_tensor * cur = attn_mix.cur; cur = build_layer_attn(inp_hyb->get_attn(), mctx_hyb, inp_kpool, cur, inp_pos, sections, il); - res_hc = build_hc_combine(res_hc, cur, inject, il); + res_hc = build_hc_combine(res_hc, cur, attn_mix.inject, il); - cur = build_hc_mix(res_hc, layer.hc_ffn_norm, layer.hc_ffn_down, layer.hc_ffn_up, layer.hc_ffn_inject, layer.hc_ffn_down_inject, &inject, il); + hc_mix ffn_mix = build_hc_mix(res_hc, layer.hc_ffn_norm, layer.hc_ffn_down, layer.hc_ffn_up, layer.hc_ffn_inject, layer.hc_ffn_down_inject, il); + cur = ffn_mix.cur; cur = build_layer_ffn(cur, il); - res_hc = build_hc_combine(res_hc, cur, inject, il); + res_hc = build_hc_combine(res_hc, cur, ffn_mix.inject, il); // the next draft step reads this residual as its h ggml_tensor * flat = ggml_reshape_2d(ctx0, res_hc, n_embd*hc, n_tokens); @@ -657,9 +661,10 @@ llama_model_qwen4exp::graph_mtp::graph_mtp(const llama_model & model, const llm_ cb(res->t_h_nextn, "h_nextn", il); ggml_build_forward_expand(gf, res->t_h_nextn); - cur = build_hc_mix(ggml_reshape_3d(ctx0, flat_out, n_embd, hc, flat_out->ne[1]), + hc_mix head_mix = build_hc_mix(ggml_reshape_3d(ctx0, flat_out, n_embd, hc, flat_out->ne[1]), layer.nextn.hc_head_norm, layer.nextn.hc_head_down, layer.nextn.hc_head_up, - nullptr, nullptr, nullptr, il); + nullptr, nullptr, il); + cur = head_mix.cur; cb(cur, "result_norm", -1); res->t_embd = cur;