From ccd360f060a0f17b87ee0d4c5d91ee84cfdff484 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 11 Nov 2026 15:16:16 +0200 Subject: [PATCH] llama : remove non-fused GDN and ubatch-dependent graph nodes (#32078) - remove build_delta_net_chunking and build_delta_net_autoregressive - always use the fused gated delta net - make graph_max_nodes independent of the ubatch size - drop the fused GDN probes and fused node tags Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4.1-Flash-1017-GGUF (Mac Studio, M5 Ultra, 256GB) --- src/llama-context.cpp | 38 +--- src/llama-context.h | 2 +- src/llama-cparams.h | 3 - src/llama-graph.h | 2 - src/models/clef.cpp | 9 +- src/models/clover-moe.cpp | 5 - src/models/delta-net-base.cpp | 405 +--------------------------------- src/models/kimi-linear.cpp | 3 +- src/models/models.h | 33 +-- src/models/qwen35.cpp | 8 - src/models/qwen35moe.cpp | 8 - src/models/qwen4exp.cpp | 7 - 12 files changed, 13 insertions(+), 510 deletions(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 670adcc02..89961847b 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -136,18 +136,6 @@ static const llm_fused_op_probe llm_fused_op_flash_attn_kv_rows_probe = { /*.n_tokens_per_seq =*/ 1, }; -static const llm_fused_op_probe llm_fused_op_gdn_ar_probe = { - /*.op =*/ LLM_FUSED_OP_GDN_AR, - /*.name =*/ "fused Gated Delta Net (autoregressive)", - /*.n_tokens_per_seq =*/ 1, -}; - -static const llm_fused_op_probe llm_fused_op_gdn_ch_probe = { - /*.op =*/ LLM_FUSED_OP_GDN_CH, - /*.name =*/ "fused Gated Delta Net (chunked)", - /*.n_tokens_per_seq =*/ 16, -}; - static const llm_fused_op_probe llm_fused_op_lid_probe = { /*.op =*/ LLM_FUSED_OP_LIGHTNING_INDEXER, /*.name =*/ "Lightning Indexer", @@ -347,10 +335,6 @@ llama_context::llama_context( cparams.auto_fa = false; } - cparams.fused_gdn_ar = true; - cparams.fused_gdn_ch = true; - cparams.auto_fgdn = false; - cparams.fused_lid = true; cparams.auto_flid = false; @@ -724,13 +708,6 @@ void llama_context::resolve_fused_ops(const llama_memory_context_i * mctx, uint3 cparams.auto_fkvr = false; } - if (cparams.auto_fgdn) { - LLAMA_LOG_INFO("%s: resolving fused Gated Delta Net support:\n", func); - resolve(llm_fused_op_gdn_ar_probe, cparams.fused_gdn_ar); - resolve(llm_fused_op_gdn_ch_probe, cparams.fused_gdn_ch); - cparams.auto_fgdn = false; - } - if (cparams.auto_flid) { LLAMA_LOG_INFO("%s: resolving fused Lightning Indexer support:\n", func); resolve(llm_fused_op_lid_probe, cparams.fused_lid); @@ -816,7 +793,7 @@ void llama_context::sched_reserve() { const uint32_t n_seqs = cparams.n_seq_max; const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch); - const size_t max_nodes = this->graph_max_nodes(n_tokens); + const size_t max_nodes = this->graph_max_nodes(); LLAMA_LOG_DEBUG("%s: max_nodes = %zu\n", __func__, max_nodes); @@ -2919,14 +2896,11 @@ void llama_context::output_reorder() { // graph // -uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const { +uint32_t llama_context::graph_max_nodes() const { uint32_t res; - if (model.arch == LLM_ARCH_KIMI_K3 || model.arch == LLM_ARCH_KIMI_K31 || model.arch == LLM_ARCH_GLM5_NEXT) { - // the n_tokens*40 budget below is exhausted at ubatch 3840 - res = std::max(n_tokens * 160, 64u * model.n_tensors()); - } else if (model.arch == LLM_ARCH_HRM_TEXT) { - // the 128-slot looped graph needs roughly one stack per token budget - res = std::max(n_tokens * 80, 64u * model.n_tensors()); + if (model.arch == LLM_ARCH_KIMI_K3 || model.arch == LLM_ARCH_KIMI_K31 || model.arch == LLM_ARCH_GLM5_NEXT || + model.arch == LLM_ARCH_HRM_TEXT) { + res = std::max(1024u, 64u * model.n_tensors()); } else if (model.arch == LLM_ARCH_QWEN3NEXT || model.arch == LLM_ARCH_KIMI_LINEAR || model.arch == LLM_ARCH_METIS16 || @@ -2944,7 +2918,7 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const { model.arch == LLM_ARCH_MINIMAX_01 || model.arch == LLM_ARCH_MINIMAX_M3 || model.arch == LLM_ARCH_HY_V4) { - res = std::max(n_tokens * 40, 32u * model.n_tensors()); + res = std::max(1024u, 32u * model.n_tensors()); } else if (model.arch == LLM_ARCH_DFLASH && model.hparams.dflash_selector_rank > 0) { // DFlash2's convolutions and selector are shape work rather than matmuls, // so they cost ~8.6 nodes per tensor against ~5.9 for a plain DFlash draft diff --git a/src/llama-context.h b/src/llama-context.h index f4061432f..d7f1123bb 100644 --- a/src/llama-context.h +++ b/src/llama-context.h @@ -272,7 +272,7 @@ private: // public: - uint32_t graph_max_nodes(uint32_t n_tokens) const; + uint32_t graph_max_nodes() const; // can reuse the llm_graph_result instance of the context (for example to update a memory module) llm_graph_result * get_gf_res_reserve() const; diff --git a/src/llama-cparams.h b/src/llama-cparams.h index 7ac1513ef..2b38e73d5 100644 --- a/src/llama-cparams.h +++ b/src/llama-cparams.h @@ -39,9 +39,6 @@ struct llama_cparams { bool offload_kqv; bool flash_attn; bool auto_fa; - bool fused_gdn_ar; // use fused gated delta net (autoregressive) - bool fused_gdn_ch; // use fused gated delta net (chunked) - bool auto_fgdn; bool fused_lid; // use fused lightning indexer bool auto_flid; bool fused_dsv4_hc_pre; diff --git a/src/llama-graph.h b/src/llama-graph.h index 636a1c54b..8242ebb89 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -49,8 +49,6 @@ enum llm_graph_type { enum llm_fused_op { LLM_FUSED_OP_FLASH_ATTN, LLM_FUSED_OP_FLASH_ATTN_KV_ROWS, - LLM_FUSED_OP_GDN_AR, - LLM_FUSED_OP_GDN_CH, LLM_FUSED_OP_LIGHTNING_INDEXER, LLM_FUSED_OP_DSV4_HC_PRE, LLM_FUSED_OP_DSV4_HC_COMB, diff --git a/src/models/clef.cpp b/src/models/clef.cpp index 3ac5782a8..86fed4e87 100644 --- a/src/models/clef.cpp +++ b/src/models/clef.cpp @@ -457,16 +457,9 @@ ggml_tensor * llama_model_clef::graph::build_layer_attn_linear( q_conv = build_gdn_l2_norm(ctx0, q_conv, hparams.f_norm_rms_eps); k_conv = build_gdn_l2_norm(ctx0, k_conv, hparams.f_norm_rms_eps); - // note: need explicit repeat only if we are not using the fused GDN - if (num_k_heads != num_v_heads && (!cparams.fused_gdn_ar || !cparams.fused_gdn_ch)) { - GGML_ASSERT(num_v_heads % num_k_heads == 0); - q_conv = ggml_repeat_4d(ctx0, q_conv, head_k_dim, num_v_heads, n_tokens, n_seqs); - k_conv = ggml_repeat_4d(ctx0, k_conv, head_k_dim, num_v_heads, n_tokens, n_seqs); - } - ggml_tensor * state = ggml_fill(ctx0, ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, head_v_dim, head_v_dim, num_v_heads, n_seqs), 0.0f); - ggml_tensor * output = build_delta_net(q_conv, k_conv, v_conv, gate, beta, state, il).first; + ggml_tensor * output = build_delta_net(q_conv, k_conv, v_conv, gate, beta, state).first; cb(output, "attn_output", il); // gated normalization diff --git a/src/models/clover-moe.cpp b/src/models/clover-moe.cpp index 9caaa5630..8892521ee 100644 --- a/src/models/clover-moe.cpp +++ b/src/models/clover-moe.cpp @@ -369,11 +369,6 @@ ggml_tensor * llama_model_clover_moe::graph::build_layer_attn_linear( q_conv = build_gdn_l2_norm(ctx0, q_conv, 1e-6f); k_conv = build_gdn_l2_norm(ctx0, k_conv, 1e-6f); - if (num_k_heads != num_v_heads && (!cparams.fused_gdn_ar || !cparams.fused_gdn_ch)) { - q_conv = ggml_repeat_4d(ctx0, q_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs); - k_conv = ggml_repeat_4d(ctx0, k_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs); - } - ggml_tensor * output = build_recurrent_attn(inp, ssm_states_all, q_conv, k_conv, v_conv, gate, beta, state, il); ggml_tensor * z_2d = ggml_reshape_4d(ctx0, z, head_v_dim, num_v_heads, n_seq_tokens, n_seqs); diff --git a/src/models/delta-net-base.cpp b/src/models/delta-net-base.cpp index 4c0c24d6c..45b2de4bb 100644 --- a/src/models/delta-net-base.cpp +++ b/src/models/delta-net-base.cpp @@ -3,378 +3,15 @@ #include "llama-impl.h" #include "llama-memory-recurrent.h" -// utility to get one slice from the third dimension -// input dim: [x, y, c, b] -// output dim: [x, y, 1, b] -static ggml_tensor * get_slice_2d(ggml_context * ctx0, ggml_tensor * t, int64_t c) { - return ggml_view_4d(ctx0, t, t->ne[0], t->ne[1], 1, t->ne[3], - t->nb[1], t->nb[2], t->nb[3], t->nb[2] * c); -} - llm_build_delta_net_base::llm_build_delta_net_base(const llm_graph_params & params) : llm_graph_context(params) {} -std::pair llm_build_delta_net_base::build_delta_net_chunking( +std::pair llm_build_delta_net_base::build_delta_net( ggml_tensor * q, ggml_tensor * k, ggml_tensor * v, ggml_tensor * g, ggml_tensor * b, - ggml_tensor * s, - int il) { - const int64_t S_k = q->ne[0]; - const int64_t H_k = q->ne[1]; - const int64_t n_tokens = q->ne[2]; - const int64_t n_seqs = q->ne[3]; - - const int64_t S_v = v->ne[0]; - const int64_t H_v = v->ne[1]; - const bool kda = (g->ne[0] == S_k && g->ne[1] == H_k); - - GGML_ASSERT(S_k == S_v); - GGML_ASSERT(H_v % H_k == 0); - - GGML_ASSERT(q->ne[0] == S_k && q->ne[1] == H_k && q->ne[2] == n_tokens && q->ne[3] == n_seqs); - GGML_ASSERT(k->ne[0] == S_k && k->ne[1] == H_k && k->ne[2] == n_tokens && k->ne[3] == n_seqs); - GGML_ASSERT(v->ne[0] == S_v && v->ne[1] == H_v && v->ne[2] == n_tokens && v->ne[3] == n_seqs); - - GGML_ASSERT(g->ne[0] == 1 || g->ne[0] == S_v); - GGML_ASSERT( g->ne[1] == H_v && g->ne[2] == n_tokens && g->ne[3] == n_seqs); - GGML_ASSERT(b->ne[0] == 1 && b->ne[1] == H_v && b->ne[2] == n_tokens && b->ne[3] == n_seqs); - GGML_ASSERT(s->ne[0] == S_v && s->ne[1] == S_v && s->ne[2] == H_v && s->ne[3] == n_seqs); - - const float scale = 1.0f / sqrtf(S_k); - - q = ggml_scale(ctx0, q, scale); - - cb(q, "q_in", il); - cb(k, "k_in", il); - cb(v, "v_in", il); - cb(b, "b_in", il); - cb(g, "g_in", il); - - q = ggml_permute(ctx0, q, 0, 2, 1, 3); // [S_k, n_tokens, H_k, n_seqs] - k = ggml_permute(ctx0, k, 0, 2, 1, 3); // [S_k, n_tokens, H_k, n_seqs] - v = ggml_permute(ctx0, v, 0, 2, 1, 3); // [S_v, n_tokens, H_v, n_seqs] - g = ggml_permute(ctx0, g, 0, 2, 1, 3); // [g_0, n_tokens, H_v, n_seqs] - b = ggml_permute(ctx0, b, 0, 2, 1, 3); // [ 1, n_tokens, H_v, n_seqs] - - const int CS = kda ? 16 : 64; // chunk size - - const int pad = (CS - n_tokens % CS) % CS; - const int n_chunks = (n_tokens + pad) / CS; - - q = ggml_pad(ctx0, q, 0, pad, 0, 0); - k = ggml_pad(ctx0, k, 0, pad, 0, 0); - v = ggml_pad(ctx0, v, 0, pad, 0, 0); - g = ggml_pad(ctx0, g, 0, pad, 0, 0); - b = ggml_pad(ctx0, b, 0, pad, 0, 0); - - ggml_tensor * v_b = ggml_mul(ctx0, v, b); - ggml_tensor * k_b = ggml_mul(ctx0, k, b); - - cb(v_b, "v_b", il); - cb(k_b, "k_b", il); - - q = ggml_reshape_4d(ctx0, q, S_k, CS, n_chunks, H_k * n_seqs); - k = ggml_reshape_4d(ctx0, k, S_k, CS, n_chunks, H_k * n_seqs); - k_b = ggml_reshape_4d(ctx0, k_b, S_k, CS, n_chunks, H_v * n_seqs); - v = ggml_reshape_4d(ctx0, v, S_v, CS, n_chunks, H_v * n_seqs); - v_b = ggml_reshape_4d(ctx0, v_b, S_v, CS, n_chunks, H_v * n_seqs); - - g = ggml_reshape_4d(ctx0, g, g->ne[0], CS, n_chunks, H_v * n_seqs); - b = ggml_reshape_4d(ctx0, b, 1, CS, n_chunks, H_v * n_seqs); - - // [CS, g_0, n_chunks, H_v * n_seqs] - // TODO: extend ggml_cumsum with axis parameter to avoid transpose - ggml_tensor * g_cs = ggml_cumsum(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, g))); - cb(g_cs, "g_cs", il); - - ggml_tensor * kb = nullptr; - ggml_tensor * kq = nullptr; - if (kda) { - const int64_t CHB = n_chunks * H_k * n_seqs; - - ggml_tensor * g_cs_i = ggml_reshape_4d(ctx0, g_cs, CS, 1, S_k, CHB); // [chunk_size, 1, S_k, CHB] - ggml_tensor * g_cs_j = ggml_reshape_4d(ctx0, g_cs, 1, CS, S_k, CHB); // [1, chunk_size, S_k, CHB] - - g_cs_j = ggml_repeat_4d(ctx0, g_cs_j, CS, CS, S_k, CHB); // [1, chunk_size, S_k, CHB] -> [chunk_size, chunk_size, S_k, CHB] - - // decay_mask [chunk_size,chunk_size,S_k,CHB] - ggml_tensor * decay_mask; - decay_mask = ggml_sub(ctx0, g_cs_j, g_cs_i); - decay_mask = ggml_tri(ctx0, decay_mask, GGML_TRI_TYPE_LOWER_DIAG); - decay_mask = ggml_exp(ctx0, decay_mask); - cb(decay_mask, "decay_mask", il); - - // decay_mask [S_k,BT_j,BT_i,CHB] *Note* second and third chunk_sizes are switched - decay_mask = ggml_cont_4d(ctx0, ggml_permute(ctx0, decay_mask, 2, 1, 0, 3), S_k, CS, CS, CHB); - - ggml_tensor * k_b_i = ggml_reshape_4d(ctx0, k_b, S_k, CS, 1, CHB); - ggml_tensor * k_j = ggml_reshape_4d(ctx0, k, S_k, 1, CS, CHB); - ggml_tensor * q_i = ggml_reshape_4d(ctx0, q, S_k, CS, 1, CHB); - - ggml_tensor * decay_k_b_i = ggml_mul(ctx0, decay_mask, k_b_i); - ggml_tensor * decay_q_i = ggml_mul(ctx0, decay_mask, q_i); - - // decay_k_b_i [S,BT,BT,CHB] @ k_j [S,1,BT,CHB] = Akk [BT,1,BT,CHB] - kb = ggml_mul_mat(ctx0, decay_k_b_i, k_j); - kq = ggml_mul_mat(ctx0, decay_q_i, k_j); - - kb = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_4d(ctx0, kb, CS, CS, n_chunks, H_v * n_seqs))); - kq = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_4d(ctx0, kq, CS, CS, n_chunks, H_v * n_seqs))); - } else { - ggml_tensor * g_cs_i = g_cs; - ggml_tensor * g_cs_j = ggml_reshape_4d(ctx0, g_cs, 1, CS, n_chunks, H_v * n_seqs); - - g_cs_j = ggml_repeat_4d(ctx0, g_cs_j, CS, CS, n_chunks, H_v * n_seqs); - - // [CS, CS, n_chunks, H_v * n_seqs] - ggml_tensor * decay_mask; - decay_mask = ggml_sub(ctx0, g_cs_j, g_cs_i); - decay_mask = ggml_tri(ctx0, decay_mask, GGML_TRI_TYPE_LOWER_DIAG); - decay_mask = ggml_exp(ctx0, decay_mask); - cb(decay_mask, "decay_mask", il); - - // [CS, CS, n_chunks, H_k * n_seqs] - kb = ggml_mul_mat(ctx0, k, k_b); - kb = ggml_mul (ctx0, kb, decay_mask); - - // [CS, CS, n_chunks, H_k * n_seqs] - kq = ggml_mul_mat(ctx0, k, q); - kq = ggml_mul(ctx0, kq, decay_mask); - } - - kq = ggml_tri(ctx0, kq, GGML_TRI_TYPE_LOWER_DIAG); - cb(kq, "kq", il); - - // [CS, CS, n_chunks, H_k * n_seqs] - ggml_tensor * attn; - attn = ggml_tri(ctx0, kb, GGML_TRI_TYPE_LOWER); - cb(attn, "attn", il); - - ggml_tensor * identity; - identity = ggml_view_1d(ctx0, attn, CS, 0); - identity = ggml_fill (ctx0, identity, 1.0f); - identity = ggml_diag (ctx0, identity); - - ggml_tensor * lhs = ggml_add(ctx0, attn, identity); - cb(lhs, "dnet_add_ch_lhs", il); - - attn = ggml_neg(ctx0, attn); - cb(attn, "attn_pre_solve", il); - - ggml_tensor * lin_solve = ggml_solve_tri(ctx0, lhs, attn, true, true, false); - attn = ggml_add(ctx0, lin_solve, identity); - cb(attn, "dnet_add_ch_attn_solved", il); // [CS, CS, n_chunks, H_k * n_seqs] - - // [S_v, CS, n_chunks, H_v * n_seqs] - v = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, v_b)), attn); - - // [CS, 1, n_chunks, H_v * n_seqs] KDA: [CS, S_k, n_chunks, H_v * n_seqs] - ggml_tensor * g_exp = ggml_exp(ctx0, g_cs); - - k_b = ggml_cont(ctx0, ggml_transpose(ctx0, k_b)); - - // [CS, S_k, n_chunks, H_k * n_seqs] - ggml_tensor * kbg = ggml_mul(ctx0, k_b, g_exp); - cb(kbg, "k_beta_g_exp", il); - - // [S_k, CS, n_chunks, H_k * n_seqs] - ggml_tensor * k_cd = ggml_mul_mat(ctx0, kbg, attn); - cb(k_cd, "k_cumdecay", il); - - // [1, CS, n_chunks, H_k * n_seqs] KDA: [S_k, CS, n_chunks, H_k * n_seqs] - ggml_tensor * g_exp_t = ggml_cont(ctx0, ggml_transpose(ctx0, g_exp)); - ggml_tensor * q_g_exp = ggml_mul(ctx0, q, g_exp_t); - - // vectorized calculation of key_gdiff - // improved from the chunked version: - // g_last = torch.clamp(g_cum[:, :, -1], max=50.0).exp().unsqueeze(-1).unsqueeze(-1) - // g_diff = torch.clamp(g_cum[:, :, -1:] - g_cum, max=50.0).exp() - // key_gdiff = key * g_diff.unsqueeze(-1) - // kgdmulvnew = (key_gdiff).transpose(-1, -2) @ v_new - // last_recurrent_state = last_recurrent_state * g_last + kgdmulvnew - - // get last element in g_cumsum along CS dimension (ne0) - // example: [[x, y, z, ..., last], ...] -> [[last], ...] - // [1, 1, n_chunks, H_v * n_seqs] KDA: [1, S_k, n_chunks, H_v * n_seqs] - ggml_tensor * g_last = ggml_view_4d(ctx0, g_cs, 1, g_cs->ne[1], g_cs->ne[2], g_cs->ne[3], - g_cs->nb[1], - g_cs->nb[2], - g_cs->nb[3], - ggml_row_size(g_cs->type, g_cs->ne[0] - 1)); - cb(g_last, "g_last", il); - - // [1, 1, n_chunks, H_v * n_seqs] KDA: [S_k, 1, n_chunks, H_v * n_seqs] - ggml_tensor * g_last_exp_t = ggml_transpose(ctx0, ggml_exp(ctx0, g_last)); - cb(g_last_exp_t, "g_last_exp_t", il); - - // [CS, 1, n_chunks, H_v * n_seqs] KDA: [CS, S_k, n_chunks, H_v * n_seqs] - ggml_tensor * g_diff = ggml_neg(ctx0, ggml_sub(ctx0, g_cs, g_last)); - cb(g_diff, "g_diff", il); - - ggml_tensor * g_diff_exp_t = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_exp(ctx0, g_diff))); - - // [S_k, CS, n_chunks, H_v * n_seqs] - ggml_tensor * kg = ggml_mul(ctx0, k, g_diff_exp_t); - cb(kg, "key_gdiff", il); - - // [CS, S_k, n_chunks, H_v * n_seqs] - ggml_tensor * kg_t = ggml_cont(ctx0, ggml_transpose(ctx0, kg)); - cb(kg_t, "key_gdiff_t", il); - - s = ggml_reshape_4d(ctx0, s, S_v, S_v, 1, H_v * n_seqs); - cb(s, "dnet_add_ch_state", il); - - // [CS, S_v, n_chunks, H_v * n_seqs] - ggml_tensor * v_t = ggml_cont(ctx0, ggml_transpose(ctx0, v)); - - for (int64_t chunk = 0; chunk < n_chunks; chunk++) { - ggml_tensor * ch_k_cd = get_slice_2d(ctx0, k_cd, chunk); // [S_k, CS, 1, H_k * n_seqs] - ggml_tensor * ch_v_t = get_slice_2d(ctx0, v_t, chunk); // [ CS, S_v, 1, H_v * n_seqs] - ggml_tensor * ch_kq = get_slice_2d(ctx0, kq, chunk); // [ CS, CS, 1, H_k * n_seqs] - ggml_tensor * ch_q_g_exp = get_slice_2d(ctx0, q_g_exp, chunk); // [S_k, CS, 1, H_k * n_seqs] - ggml_tensor * ch_kg_t = get_slice_2d(ctx0, kg_t, chunk); // [ CS, S_k, 1, H_v * n_seqs] - - // [CS, S_v, 1, H_v * n_seqs] - ggml_tensor * v_t_p = ggml_mul_mat(ctx0, ch_k_cd, s); - cb(v_t_p, "v_prime", il); - - // [CS, S_v, 1, H_v * n_seqs] - ggml_tensor * v_t_new = ggml_sub(ctx0, ch_v_t, v_t_p); - cb(v_t_new, "v_t_new", il); - - // [S_v, CS, 1, H_v * n_seqs] - ggml_tensor * v_attn = ggml_mul_mat(ctx0, v_t_new, ch_kq); - cb(v_attn, "v_attn", il); - - // [S_v, CS, 1, H_v * n_seqs] - ggml_tensor * attn_inter = ggml_mul_mat(ctx0, s, ch_q_g_exp); - cb(attn_inter, "attn_inter", il); - - // [S_v, CS, 1, H_v * n_seqs] - ggml_tensor * o_ch = ggml_add(ctx0, attn_inter, v_attn); - cb(o_ch, "dnet_add_ch_attn_out", il); - - v = ggml_set_inplace(ctx0, v, o_ch, v->nb[1], v->nb[2], v->nb[3], chunk * v->nb[2]); - - // kgdmulvnew = (key_gdiff).transpose(-1, -2) @ v_new - // TODO: head broadcast might not work here - probably will need a transpose - ggml_tensor * kgv = ggml_mul_mat(ctx0, ch_kg_t, v_t_new); // [S_k, S_v, 1, H_k * n_seqs] - - // last_recurrent_state = last_recurrent_state * g_last + kgdmulvnew - ggml_tensor * ch_g_last_exp_t = get_slice_2d(ctx0, g_last_exp_t, chunk); - - s = ggml_mul(ctx0, s, ch_g_last_exp_t); - s = ggml_add(ctx0, s, kgv); - cb(s, "dnet_add_ch_state", il); - } - - // truncate padded tokens - ggml_tensor * o = ggml_view_4d(ctx0, v, - S_v, n_tokens, H_v, n_seqs, - ggml_row_size(v->type, S_v), - ggml_row_size(v->type, S_v * CS * n_chunks), - ggml_row_size(v->type, S_v * CS * n_chunks * H_v), 0); - o = ggml_permute (ctx0, o, 0, 2, 1, 3); // [S_v, H_v, n_tokens, n_seqs] - s = ggml_reshape_4d(ctx0, s, S_v, S_v, H_v, n_seqs); - cb(s, "output_state", il); - - return {o, s}; -} - -std::pair llm_build_delta_net_base::build_delta_net_autoregressive( - ggml_tensor * q, - ggml_tensor * k, - ggml_tensor * v, - ggml_tensor * g, - ggml_tensor * b, // beta - ggml_tensor * s, // state - int il) { - const int64_t S_k = q->ne[0]; - const int64_t H_k = q->ne[1]; - const int64_t n_tokens = q->ne[2]; - const int64_t n_seqs = q->ne[3]; - - const int64_t S_v = v->ne[0]; - const int64_t H_v = v->ne[1]; - - GGML_ASSERT(n_tokens == 1); - - GGML_ASSERT(S_k == S_v); - GGML_ASSERT(H_v % H_k == 0); - - GGML_ASSERT(q->ne[0] == S_k && q->ne[1] == H_k && q->ne[2] == n_tokens && q->ne[3] == n_seqs); - GGML_ASSERT(k->ne[0] == S_k && k->ne[1] == H_k && k->ne[2] == n_tokens && k->ne[3] == n_seqs); - GGML_ASSERT(v->ne[0] == S_v && v->ne[1] == H_v && v->ne[2] == n_tokens && v->ne[3] == n_seqs); - - GGML_ASSERT(g->ne[0] == 1 || g->ne[0] == S_v); - GGML_ASSERT( g->ne[1] == H_v && g->ne[2] == n_tokens && g->ne[3] == n_seqs); - GGML_ASSERT(b->ne[0] == 1 && b->ne[1] == H_v && b->ne[2] == n_tokens && b->ne[3] == n_seqs); - GGML_ASSERT(s->ne[0] == S_v && s->ne[1] == S_v && s->ne[2] == H_v && s->ne[3] == n_seqs); - - const float scale = 1.0f / sqrtf(S_k); - - q = ggml_scale(ctx0, q, scale); - - q = ggml_permute(ctx0, q, 0, 2, 1, 3); // [S_k, n_tokens, H_k, n_seqs] - k = ggml_permute(ctx0, k, 0, 2, 1, 3); // [S_k, n_tokens, H_k, n_seqs] - v = ggml_permute(ctx0, v, 0, 2, 1, 3); // [S_v, n_tokens, H_v, n_seqs] - - cb(q, "q_in", il); - cb(k, "k_in", il); - cb(v, "v_in", il); - cb(b, "b_in", il); - cb(g, "g_in", il); - - // GDA: [1, 1, H_v, n_seqs] - // KDA: [1, S_k, H_v, n_seqs] - g = ggml_reshape_4d(ctx0, g, 1, g->ne[0], H_v, n_seqs); - b = ggml_reshape_4d(ctx0, b, 1, 1, H_v, n_seqs); - - // [S_v, S_v, H_v, n_seqs] - g = ggml_exp(ctx0, g); - s = ggml_mul(ctx0, s, g); - - // [1, S_v, H_v, n_seqs] - ggml_tensor * sk; - sk = ggml_mul (ctx0, s, k); - sk = ggml_sum_rows(ctx0, sk); - - // [S_v, 1, H_v, n_seqs] - ggml_tensor * d; - d = ggml_sub(ctx0, v, ggml_transpose(ctx0, sk)); - d = ggml_mul(ctx0, d, b); - - // [1, S_v, H_v, n_seqs] - ggml_tensor * d_t; - d_t = ggml_transpose(ctx0, d); - - // [S_v, S_v, H_v, n_seqs] - ggml_tensor * kd; - k = ggml_repeat(ctx0, k, s); - kd = ggml_mul (ctx0, k, d_t); - - s = ggml_add(ctx0, s, kd); - - cb(s, "dnet_add_ar_state", il); - - ggml_tensor * s_q = ggml_mul (ctx0, s, q); - ggml_tensor * o = ggml_sum_rows(ctx0, s_q); - - o = ggml_permute (ctx0, o, 2, 0, 1, 3); // [S_v, H_v, n_tokens, n_seqs] - - return {o, s}; -} - -std::pair llm_build_delta_net_base::build_delta_net_fused( - ggml_tensor * q, - ggml_tensor * k, - ggml_tensor * v, - ggml_tensor * g, - ggml_tensor * b, - ggml_tensor * s, - int il) { + ggml_tensor * s) { const int64_t S_k = q->ne[0]; const int64_t H_k = q->ne[1]; const int64_t n_tokens = q->ne[2]; @@ -397,12 +34,6 @@ std::pair llm_build_delta_net_base::build_delta_ne // K=1: output carries the final state only. state s is 4D [S_v, S_v, H_v, n_seqs]. ggml_tensor * result = ggml_gated_delta_net(ctx0, q, k, v, g, b, s, /*K=*/1); - if (n_tokens == 1) { - res->add_fused_node({LLM_FUSED_OP_GDN_AR, result, il}); - } else { - res->add_fused_node({LLM_FUSED_OP_GDN_CH, result, il}); - } - ggml_tensor * output = ggml_view_4d(ctx0, result, S_v, H_v, n_tokens, n_seqs, ggml_row_size(result->type, S_v), @@ -419,30 +50,6 @@ std::pair llm_build_delta_net_base::build_delta_ne return {output, new_state}; } -std::pair llm_build_delta_net_base::build_delta_net( - ggml_tensor * q, - ggml_tensor * k, - ggml_tensor * v, - ggml_tensor * g, - ggml_tensor * b, - ggml_tensor * s, - int il) { - const int64_t n_seq_tokens = q->ne[2]; - - if (n_seq_tokens == 1) { - if (cparams.fused_gdn_ar) { - return build_delta_net_fused(q, k, v, g, b, s, il); - } - return build_delta_net_autoregressive(q, k, v, g, b, s, il); - } - - if (cparams.fused_gdn_ch) { - return build_delta_net_fused(q, k, v, g, b, s, il); - } - - return build_delta_net_chunking(q, k, v, g, b, s, il); -} - ggml_tensor * llm_build_delta_net_base::build_conv_state( llm_graph_input_rs * inp, ggml_tensor * conv_states_all, @@ -491,7 +98,7 @@ ggml_tensor * llm_build_delta_net_base::build_recurrent_attn( const bool keep = cparams.n_rs_seq > 0; if (!keep) { - auto attn_out = build_delta_net(q, k, v, g, b, s, il); + auto attn_out = build_delta_net(q, k, v, g, b, s); ggml_tensor * output = attn_out.first; ggml_tensor * new_state = attn_out.second; cb(output, "attn_output", il); @@ -510,12 +117,6 @@ ggml_tensor * llm_build_delta_net_base::build_recurrent_attn( // state s is 4D [S_v, S_v, H_v, n_seqs]; K snapshot slots are written into the output. ggml_tensor * gdn_out = ggml_gated_delta_net(ctx0, q, k, v, g, b, s, K); - if (n_seq_tokens > 1) { - res->add_fused_node({LLM_FUSED_OP_GDN_CH, gdn_out, il}); - } else { - res->add_fused_node({LLM_FUSED_OP_GDN_AR, gdn_out, il}); - } - const int64_t attn_score_elems = S_v * H_v * n_seq_tokens * n_seqs; const int64_t state_size_per_snap = S_v * S_v * H_v * n_seqs; diff --git a/src/models/kimi-linear.cpp b/src/models/kimi-linear.cpp index 54d30ad47..2190df8c9 100644 --- a/src/models/kimi-linear.cpp +++ b/src/models/kimi-linear.cpp @@ -348,8 +348,7 @@ llama_model_kimi_linear::graph::graph(const llm_graph_params & params) : Qcur = build_gdn_l2_norm(ctx0, Qcur, eps_norm); Kcur = build_gdn_l2_norm(ctx0, Kcur, eps_norm); - // Choose between build_delta_net_chunking and build_delta_net_recurrent based on n_tokens - auto attn_out = build_delta_net(Qcur, Kcur, Vcur, g1, beta, state, il); + auto attn_out = build_delta_net(Qcur, Kcur, Vcur, g1, beta, state); ggml_tensor * output = ggml_cont(ctx0, attn_out.first); ggml_tensor * new_state = attn_out.second; diff --git a/src/models/models.h b/src/models/models.h index f2c43b800..cfef363b6 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -37,44 +37,13 @@ struct llm_build_delta_net_base : public llm_graph_context { virtual ~llm_build_delta_net_base() = default; // returns pair of output and new state - std::pair build_delta_net_chunking( - ggml_tensor * q, - ggml_tensor * k, - ggml_tensor * v, - ggml_tensor * g, - ggml_tensor * b, - ggml_tensor * s, - int il); - - // returns pair of output and new state - std::pair build_delta_net_autoregressive( - ggml_tensor * q, - ggml_tensor * k, - ggml_tensor * v, - ggml_tensor * g, - ggml_tensor * b, - ggml_tensor * s, - int il); - - // use the ggml_gated_delta_net fused operator (K=1; state has shape [S_v, S_v, H_v, n_seqs]) - std::pair build_delta_net_fused( - ggml_tensor * q, - ggml_tensor * k, - ggml_tensor * v, - ggml_tensor * g, - ggml_tensor * b, - ggml_tensor * s, - int il); - - // choose one of two implementations above based on the number of tokens std::pair build_delta_net( ggml_tensor * q, ggml_tensor * k, ggml_tensor * v, ggml_tensor * g, ggml_tensor * b, - ggml_tensor * s, - int il); + ggml_tensor * s); // read conv state from cache, concat with qkv_mixed, write back (single slot or per-token) // qkv_mixed: (qkv_dim, n_seq_tokens, n_seqs); returns conv_input: (kernel_size + n_seq_tokens - 1, channels, n_seqs) diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index 736e23f10..6ec3cff44 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -432,14 +432,6 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn_linear( //k_conv = ggml_cont_4d(ctx0, k_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs); //v_conv = ggml_cont_4d(ctx0, v_conv, head_v_dim, num_v_heads, n_seq_tokens, n_seqs); - // if head keys and value keys are different, repeat to force tensors into matching shapes - // note: need explicit repeat only if we are not using the fused GDN. - if (num_k_heads != num_v_heads && (!cparams.fused_gdn_ar || !cparams.fused_gdn_ch)) { - GGML_ASSERT(num_v_heads % num_k_heads == 0); - q_conv = ggml_repeat_4d(ctx0, q_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs); - k_conv = ggml_repeat_4d(ctx0, k_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs); - } - cb(q_conv, "q_conv_predelta", il); cb(k_conv, "k_conv_predelta", il); cb(v_conv, "v_conv_predelta", il); diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cpp index c4da3e2be..c0ee6fa03 100644 --- a/src/models/qwen35moe.cpp +++ b/src/models/qwen35moe.cpp @@ -442,14 +442,6 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn_linear( //k_conv = ggml_cont_4d(ctx0, k_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs); //v_conv = ggml_cont_4d(ctx0, v_conv, head_v_dim, num_v_heads, n_seq_tokens, n_seqs); - // if head keys and value keys are different, repeat to force tensors into matching shapes - // note: need explicit repeat only if we are not using the fused GDN. - if (num_k_heads != num_v_heads && (!cparams.fused_gdn_ar || !cparams.fused_gdn_ch)) { - GGML_ASSERT(num_v_heads % num_k_heads == 0); - q_conv = ggml_repeat_4d(ctx0, q_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs); - k_conv = ggml_repeat_4d(ctx0, k_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs); - } - cb(q_conv, "q_conv_predelta", il); cb(k_conv, "k_conv_predelta", il); cb(v_conv, "v_conv_predelta", il); diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index 876640e16..181e4dca4 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -1191,13 +1191,6 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn_linear( q_conv = build_gdn_l2_norm(ctx0, q_conv, eps_norm); k_conv = build_gdn_l2_norm(ctx0, k_conv, eps_norm); - // repeat to match shapes when head keys != value keys; unneeded with the fused GDN - if (num_k_heads != num_v_heads && (!cparams.fused_gdn_ar || !cparams.fused_gdn_ch)) { - GGML_ASSERT(num_v_heads % num_k_heads == 0); - q_conv = ggml_repeat_4d(ctx0, q_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs); - k_conv = ggml_repeat_4d(ctx0, k_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs); - } - cb(q_conv, "q_conv_predelta", il); cb(k_conv, "k_conv_predelta", il); cb(v_conv, "v_conv_predelta", il);