From ca8daad199a9c12c5b54643b974c640939a0f7bd Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 3 Nov 2026 08:11:06 +0200 Subject: [PATCH] llama : drop the redundant cnv from llm_attn_mask (#31558) The dspark block mask is now derived at input-fill time: llm_attn_mask::block_uniform fills every token of a sequence with the mask row of its first token, so the graph-level reshape/repeat in dflash is gone and cnv always aliases raw. Assisted-by: dsh:llama.cpp/ggml-org/GLM-5.3-Flash-GGUF (Mac Studio, M5 Ultra, 256GB) --- src/llama-graph.cpp | 37 +++++++++-------------------------- src/llama-graph.h | 5 +++-- src/llama-kv-cache.cpp | 18 ++++++++++++++--- src/models/clef.cpp | 2 -- src/models/deepseek32.cpp | 4 ++-- src/models/deepseek4.cpp | 6 +++--- src/models/deepseek41.cpp | 2 +- src/models/dflash.cpp | 22 ++++----------------- src/models/dots3note.cpp | 4 ++-- src/models/glm-dsa.cpp | 4 ++-- src/models/hy-v4.cpp | 4 ++-- src/models/lfm2.cpp | 1 - src/models/minimax-m3.cpp | 2 +- src/models/mistral-large4.cpp | 4 ++-- 14 files changed, 46 insertions(+), 69 deletions(-) diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index fe8bdebe5..f0463f849 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -124,8 +124,6 @@ static llm_attn_mask build_attn_inp_attn_mask( ggml_set_name(attn_mask.raw, "attn_inp_kq_mask"); } - attn_mask.cnv = attn_mask.raw; - return attn_mask; } @@ -3131,16 +3129,11 @@ llm_graph_input_attn_no_cache * llm_graph_context::build_attn_inp_no_cache() con ggml_set_input(inp->self_attn_mask.raw); cb(inp->self_attn_mask.raw, "self_kq_mask", -1); - inp->self_attn_mask.cnv = inp->self_attn_mask.raw; - if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) { inp->self_attn_mask_swa.raw = ggml_new_tensor_4d(ctx0, type_mask, n_kv, n_tokens, 1, 1); ggml_set_input(inp->self_attn_mask_swa.raw); - - inp->self_attn_mask_swa.cnv = inp->self_attn_mask_swa.raw; } else { inp->self_attn_mask_swa.raw = nullptr; - inp->self_attn_mask_swa.cnv = nullptr; } return (llm_graph_input_attn_no_cache *) res->add_input(std::move(inp)); @@ -3169,7 +3162,7 @@ ggml_tensor * llm_graph_context::build_attn( const bool is_swa = hparams.is_swa(il); - const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.cnv : inp->self_attn_mask.cnv; + const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.raw : inp->self_attn_mask.raw; const auto & kv_rows = is_swa ? inp->self_attn_mask_swa.rows : inp->self_attn_mask.rows; // [TAG_NO_CACHE_PAD] @@ -3296,7 +3289,7 @@ ggml_tensor * llm_graph_context::build_attn( v = mctx_cur->get_v(ctx0, il); } - const auto & kq_mask = inp->self_attn_mask.cnv; + const auto & kq_mask = inp->self_attn_mask.raw; const auto & kv_rows = inp->self_attn_mask.rows; ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, kv_rows, sinks, v_mla, 0, kq_scale, il); @@ -3341,7 +3334,6 @@ static std::unique_ptr build_attn_inp_k_impl( inp->self_k_idxs = mctx_cur->build_input_k_idxs(ctx0, ubatch); inp->self_attn_mask.raw = build_attn_inp_kq_mask(ctx0, mctx_cur, ubatch, cparams); - inp->self_attn_mask.cnv = inp->self_attn_mask.raw; } return inp; @@ -3384,7 +3376,7 @@ ggml_tensor * llm_graph_context::build_attn( ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, k_cur, k_idxs, il)); } - const auto & kq_mask = inp->self_attn_mask.cnv; + const auto & kq_mask = inp->self_attn_mask.raw; const auto & kv_rows = inp->self_attn_mask.rows; ggml_tensor * q = q_cur; @@ -3465,9 +3457,9 @@ ggml_tensor * llm_graph_context::build_attn( // top_k indexes the cells, and the indexer mask of the same type is built the same way // let's use it, so when no other layer reads the MLA mask is not allocated - ggml_tensor * kq_mask = inp->self_attn_mask_mla.cnv; - if (inp->self_attn_mask_lid.cnv->type == kq_mask->type) { - kq_mask = inp->self_attn_mask_lid.cnv; + ggml_tensor * kq_mask = inp->self_attn_mask_mla.raw; + if (inp->self_attn_mask_lid.raw->type == kq_mask->type) { + kq_mask = inp->self_attn_mask_lid.raw; } ggml_tensor * kq_mask_top_k = nullptr; @@ -3589,7 +3581,7 @@ ggml_tensor * llm_graph_context::build_attn( ggml_build_forward_expand(gf, mctx_cur->cpy_v(ctx0, v_cur, v_idxs, il)); } - const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.cnv : inp->self_attn_mask.cnv; + const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.raw : inp->self_attn_mask.raw; const auto & kv_rows = is_swa ? inp->self_attn_mask_swa.rows : inp->self_attn_mask.rows; ggml_tensor * q = q_cur; @@ -3660,7 +3652,7 @@ ggml_tensor * llm_graph_context::build_attn( ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, k_cur, k_idxs, il)); } - const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.cnv : inp->self_attn_mask.cnv; + const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.raw : inp->self_attn_mask.raw; const auto & kv_rows = is_swa ? inp->self_attn_mask_swa.rows : inp->self_attn_mask.rows; // MLA-style attention: the cached K is used as V @@ -3697,8 +3689,6 @@ llm_graph_input_attn_cross * llm_graph_context::build_attn_inp_cross() const { inp->cross_attn_mask.raw = ggml_new_tensor_4d(ctx0, type_mask, n_enc, n_tokens, 1, 1); ggml_set_input(inp->cross_attn_mask.raw); - inp->cross_attn_mask.cnv = inp->cross_attn_mask.raw; - return (llm_graph_input_attn_cross *) res->add_input(std::move(inp)); } @@ -3721,7 +3711,7 @@ ggml_tensor * llm_graph_context::build_attn( ggml_build_forward_expand(gf, k_cur); ggml_build_forward_expand(gf, v_cur); - const auto & kq_mask = inp->cross_attn_mask.cnv; + const auto & kq_mask = inp->cross_attn_mask.raw; const auto & kv_rows = inp->cross_attn_mask.rows; ggml_tensor * q = q_cur; @@ -3759,7 +3749,6 @@ static std::unique_ptr build_attn_inp_k_dsa_impl( inp->self_k_idxs_mla = mctx_cur->get_mla()->build_input_k_idxs(ctx0, ubatch); inp->self_attn_mask_mla.raw = build_attn_inp_kq_mask(ctx0, mctx_cur->get_mla(), ubatch, cparams); - inp->self_attn_mask_mla.cnv = inp->self_attn_mask_mla.raw; } { @@ -3770,7 +3759,6 @@ static std::unique_ptr build_attn_inp_k_dsa_impl( cparams_copy.flash_attn = cparams.fused_lid; inp->self_attn_mask_lid.raw = build_attn_inp_kq_mask(ctx0, mctx_cur->get_lid(), ubatch, cparams_copy); - inp->self_attn_mask_lid.cnv = inp->self_attn_mask_lid.raw; inp->self_k_rot_lid = mctx_cur->get_lid()->get_k_rot(); } @@ -3797,7 +3785,6 @@ llm_graph_input_attn_k_dsa_iswa * llm_graph_context::build_attn_inp_k_dsa_iswa() inp_swa->self_k_idxs = mctx_cur->get_swa()->build_input_k_idxs(ctx0, ubatch); inp_swa->self_attn_mask.raw = build_attn_inp_kq_mask(ctx0, mctx_cur->get_swa(), ubatch, cparams); - inp_swa->self_attn_mask.cnv = inp_swa->self_attn_mask.raw; auto inp = std::make_unique(std::move(inp_dsa), std::move(inp_swa), mctx_cur); @@ -3819,7 +3806,6 @@ llm_graph_input_attn_kv_msa * llm_graph_context::build_attn_inp_kv_msa(bool msa_ inp->self_v_idxs = mctx_base->build_input_v_idxs(ctx0, ubatch); inp->self_attn_mask.raw = build_attn_inp_kq_mask(ctx0, mctx_base, ubatch, cparams); - inp->self_attn_mask.cnv = inp->self_attn_mask.raw; } inp->self_k_rot = mctx_base->get_k_rot(); @@ -3874,7 +3860,6 @@ llm_graph_input_attn_k_iswa * llm_graph_context::build_attn_inp_k_iswa() const { inp->self_k_idxs = mctx_cur->get_base()->build_input_k_idxs(ctx0, ubatch); inp->self_attn_mask.raw = build_attn_inp_kq_mask(ctx0, mctx_cur->get_base(), ubatch, cparams); - inp->self_attn_mask.cnv = inp->self_attn_mask.raw; } { @@ -3883,7 +3868,6 @@ llm_graph_input_attn_k_iswa * llm_graph_context::build_attn_inp_k_iswa() const { inp->self_k_idxs_swa = mctx_cur->get_swa()->build_input_k_idxs(ctx0, ubatch); inp->self_attn_mask_swa.raw = build_attn_inp_kq_mask(ctx0, mctx_cur->get_swa(), ubatch, cparams); - inp->self_attn_mask_swa.cnv = inp->self_attn_mask_swa.raw; } inp->self_k_rot = mctx_cur->get_base()->get_k_rot(); @@ -3905,7 +3889,6 @@ llm_graph_input_dsv4 * llm_graph_context::build_inp_dsv4() const { inp_raw->self_k_idxs = raw_ctx->build_input_k_idxs(ctx0, ubatch); inp_raw->self_attn_mask.raw = dsv4_build_raw_kq_mask(ctx0, raw_ctx, ubatch, cparams, n_stream); - inp_raw->self_attn_mask.cnv = inp_raw->self_attn_mask.raw; inp_raw->self_k_rot = raw_ctx->get_k_rot(); auto inp = std::make_unique(cparams, std::move(inp_raw), mctx_cur); @@ -4089,7 +4072,6 @@ llm_graph_input_mem_hybrid_iswa * llm_graph_context::build_inp_mem_hybrid_iswa() inp_attn->self_v_idxs = attn_ctx->get_base()->build_input_v_idxs(ctx0, ubatch); inp_attn->self_attn_mask.raw = build_attn_inp_kq_mask(ctx0, attn_ctx->get_base(), ubatch, cparams); - inp_attn->self_attn_mask.cnv = inp_attn->self_attn_mask.raw; } { @@ -4097,7 +4079,6 @@ llm_graph_input_mem_hybrid_iswa * llm_graph_context::build_inp_mem_hybrid_iswa() inp_attn->self_v_idxs_swa = attn_ctx->get_swa()->build_input_v_idxs(ctx0, ubatch); inp_attn->self_attn_mask_swa.raw = build_attn_inp_kq_mask(ctx0, attn_ctx->get_swa(), ubatch, cparams); - inp_attn->self_attn_mask_swa.cnv = inp_attn->self_attn_mask_swa.raw; } inp_attn->self_k_rot = attn_ctx->get_base()->get_k_rot(); diff --git a/src/llama-graph.h b/src/llama-graph.h index 37b47acbb..ea2cf9f87 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -333,12 +333,13 @@ public: // the mask has two shapes depending on whether `rows` is used: // rows != nullptr: [n_kv_rows, n_batch/n_slices, 1, n_slices] // rows == nullptr: [n_kv, n_batch/n_stream, 1, n_stream] -// TODO: cnv is redundant (always aliases raw) except dflash overwrites it with a derived block mask [TAG_ATTN_MASK_CNV] struct llm_attn_mask { ggml_tensor * raw = nullptr; - ggml_tensor * cnv = nullptr; // same shape ggml_tensor * rows = nullptr; // I32 [n_kv_rows, n_slices] uint32_t n_kv = 0; + + // fill every token of a sequence with the mask row of its first token (dspark draft blocks) + bool block_uniform = false; }; class llm_graph_input_attn_no_cache : public llm_graph_input_i { diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index d9f46f0ad..06e218ba6 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -1683,6 +1683,9 @@ struct args_set_input_attn_mask { // cell of each mask column per stream, nullptr if the columns are the cells const int32_t * rows; + + // every token of a sequence reuses the mask row of the first one (dspark draft blocks) + bool block_uniform; }; template @@ -1709,10 +1712,15 @@ static void set_input_attn_mask_impl(const args_set_input_attn_mask & args, T * llama_pos seq_pos_min[LLAMA_MAX_SEQ]; std::fill(seq_pos_min, seq_pos_min + LLAMA_MAX_SEQ, INT32_MAX); + // first token of each sequence, used when the mask rows are shared within a sequence + uint32_t seq_first[LLAMA_MAX_SEQ]; + std::fill(seq_first, seq_first + LLAMA_MAX_SEQ, ubatch->n_tokens); + for (uint32_t i = 0; i < ubatch->n_tokens; ++i) { const llama_seq_id seq_id = ubatch->seq_id[i][0]; seq_pos_min[seq_id] = std::min(seq_pos_min[seq_id], ubatch->pos[i]); + seq_first[seq_id] = std::min(seq_first[seq_id], i); } for (uint32_t s = 0; s < n_stream; ++s) { @@ -1729,12 +1737,15 @@ static void set_input_attn_mask_impl(const args_set_input_attn_mask & args, T * const auto & cells = v_cells.at(seq_to_stream[seq_id]); + // shared mask rows: every query of the sequence uses the row of its first token + const uint32_t iq = args.block_uniform ? seq_first[seq_id] : i; + llama_pos p0 = -1; - const llama_pos p1 = ubatch->pos[i]; + const llama_pos p1 = ubatch->pos[iq]; // for M-RoPE - const llama_pos p1_x = is_2d ? ubatch->pos[i + ubatch->n_tokens*2] : 0; - const llama_pos p1_y = is_2d ? ubatch->pos[i + ubatch->n_tokens] : 0; + const llama_pos p1_x = is_2d ? ubatch->pos[iq + ubatch->n_tokens*2] : 0; + const llama_pos p1_y = is_2d ? ubatch->pos[iq + ubatch->n_tokens] : 0; const uint64_t idst = n_kv*i; @@ -1941,6 +1952,7 @@ void llama_kv_cache::set_input_attn_mask(const llm_attn_mask & attn_mask, const /*.n_stream =*/ n_stream, /*.n_tps =*/ n_tps, /*.rows =*/ rows, + /*.block_uniform =*/ attn_mask.block_uniform, }; if (dst->type == GGML_TYPE_F16) { diff --git a/src/models/clef.cpp b/src/models/clef.cpp index e9be0ec14..89d9419b8 100644 --- a/src/models/clef.cpp +++ b/src/models/clef.cpp @@ -352,8 +352,6 @@ llm_graph_input_attn_no_cache * llama_model_clef::graph::build_attn_inp_causal() ggml_set_input(inp->self_attn_mask.raw); cb(inp->self_attn_mask.raw, "self_kq_mask", -1); - inp->self_attn_mask.cnv = inp->self_attn_mask.raw; - return (llm_graph_input_attn_no_cache *) res->add_input(std::move(inp)); } diff --git a/src/models/deepseek32.cpp b/src/models/deepseek32.cpp index 22f516a48..450022016 100644 --- a/src/models/deepseek32.cpp +++ b/src/models/deepseek32.cpp @@ -271,7 +271,7 @@ llama_model_deepseek32::graph::graph(const llama_model & model, const llm_graph_ ggml_tensor * indexer_score = nullptr; if (cparams.fused_lid) { - indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.cnv); + indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.raw); cb(indexer_score, "indexer_score", il); res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il}); } else { @@ -305,7 +305,7 @@ llama_model_deepseek32::graph::graph(const llama_model & model, const llm_graph_ cb(indexer_score, "indexer_score", il); // mask indexer scores - ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.cnv; + ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.raw; indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask); cb(indexer_score, "indexer_score", il); } diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp index 85fe141fd..1207c3bb7 100644 --- a/src/models/deepseek4.cpp +++ b/src/models/deepseek4.cpp @@ -759,7 +759,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_csa_lid_attention( ggml_tensor * k_all = ggml_concat(ctx0, raw_k, csa_k, 2); cb(k_all, "csa_k_all", il); - ggml_tensor * raw_mask = inp_attn->self_attn_mask.cnv; + ggml_tensor * raw_mask = inp_attn->self_attn_mask.raw; ggml_tensor * csa_mask = build_top_k_mask(inp_csa.kq_mask, top_k, "csa_top_k_mask", il); ggml_tensor * kq_mask = ggml_concat(ctx0, raw_mask, csa_mask, 0); @@ -815,7 +815,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_hca_attention( ggml_tensor * k_all = ggml_concat(ctx0, raw_k, hca_k, 2); cb(k_all, "hca_k_all", il); - ggml_tensor * raw_mask = inp_attn->self_attn_mask.cnv; + ggml_tensor * raw_mask = inp_attn->self_attn_mask.raw; ggml_tensor * hca_mask = inp_hca.kq_mask; ggml_tensor * kq_mask = ggml_concat(ctx0, raw_mask, hca_mask, 0); @@ -853,7 +853,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_raw_attention( ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, kv, inp_attn->get_k_idxs(), il)); - ggml_tensor * kq_mask = inp_attn->self_attn_mask.cnv; + ggml_tensor * kq_mask = inp_attn->self_attn_mask.raw; ggml_tensor * k = mctx_cur->get_k(ctx0, il); diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp index d82998970..c4ca567e7 100644 --- a/src/models/deepseek41.cpp +++ b/src/models/deepseek41.cpp @@ -642,7 +642,7 @@ ggml_tensor * llama_model_deepseek41::graph::build_attention41( ggml_build_forward_expand(gf, inp_attn->mctx->cpy_k(ctx0, kv, inp_attn->get_k_idxs(), il)); ggml_tensor * raw_k = inp_attn->mctx->get_k(ctx0, il); - ggml_tensor * raw_mask = inp_attn->self_attn_mask.cnv; + ggml_tensor * raw_mask = inp_attn->self_attn_mask.raw; const int32_t topk_src = hparams.dsv41_topk_src[il]; const int64_t k_sel = top_k->ne[0]; diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp index fdd67e017..0de2f38ec 100644 --- a/src/models/dflash.cpp +++ b/src/models/dflash.cpp @@ -1391,22 +1391,6 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_ // * embd batch -> writes every stage's kv_norm(wkv(main_proj(target features))) into its window ring // * token batch -> [anchor, noise...] block through the stages, single-pass mHC, no per-head q norm // window of a block = window of its first query, see n_swa -// TODO: move this to a new type of `llm_graph_input` and implement this in `set_input` [TAG_ATTN_MASK_CNV] -static void build_dspark_block_mask(llm_graph_context & g, llm_graph_input_attn_k_iswa * inp_attn) { - ggml_context * ctx0 = g.ctx0; - - ggml_tensor * m = inp_attn->self_attn_mask_swa.cnv; - const int64_t n_tokens = g.n_tokens; - const int64_t n_blocks = g.ubatch.n_seqs_unq; - const int64_t bd = n_blocks > 0 ? n_tokens / n_blocks : 0; - if (m && bd > 1 && n_tokens % n_blocks == 0 && m->ne[2] == 1 && m->ne[1] % bd == 0 && ggml_is_contiguous(m)) { - ggml_tensor * m4 = ggml_reshape_4d(ctx0, m, m->ne[0], bd, m->ne[1]/bd, m->ne[3]); - ggml_tensor * r0 = ggml_view_4d(ctx0, m4, m4->ne[0], 1, m4->ne[2], m4->ne[3], m4->nb[1], m4->nb[2], m4->nb[3], 0); - m4 = ggml_repeat(ctx0, r0, m4); - inp_attn->self_attn_mask_swa.cnv = ggml_reshape_4d(ctx0, m4, m->ne[0], m->ne[1], 1, m->ne[3]); - g.cb(inp_attn->self_attn_mask_swa.cnv, "dspark_block_mask", -1); - } -} llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const llm_graph_params & params) : llama_model_deepseek41::graph(params) { @@ -1496,7 +1480,8 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll inpL = ggml_repeat_4d(ctx0, inpL, n_embd, hc, n_tokens, 1); cb(inpL, "hc_init", -1); - build_dspark_block_mask(*this, inp_attn); + // the whole block shares the mask row of its first query (the anchor) + inp_attn->self_attn_mask_swa.block_uniform = true; ggml_tensor * pre_prev = nullptr; @@ -1743,7 +1728,8 @@ llama_model_dflash::graph_mistral_large4::graph_mistral_large4(const llama_model } cb(inpL, "inp_noise_embd", -1); - build_dspark_block_mask(*this, inp_attn); + // the whole block shares the mask row of its first query (the anchor) + inp_attn->self_attn_mask_swa.block_uniform = true; for (int il = 0; il < n_layer; ++il) { const auto & layer = model.layers[il]; diff --git a/src/models/dots3note.cpp b/src/models/dots3note.cpp index df6fee7e4..c848b0fe2 100644 --- a/src/models/dots3note.cpp +++ b/src/models/dots3note.cpp @@ -256,7 +256,7 @@ llama_model_dots3note::graph::graph(const llama_model & model, const llm_graph_p ggml_tensor * indexer_score = nullptr; if (cparams.fused_lid) { - indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn->get_dsa()->self_attn_mask_lid.cnv); + indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn->get_dsa()->self_attn_mask_lid.raw); cb(indexer_score, "indexer_score", il); res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il}); } else { @@ -286,7 +286,7 @@ llama_model_dots3note::graph::graph(const llama_model & model, const llm_graph_p indexer_score = ggml_cont(ctx0, ggml_permute(ctx0, indexer_score, 2, 1, 0, 3)); cb(indexer_score, "indexer_score", il); - ggml_tensor * indexer_kq_mask = inp_attn->get_dsa()->self_attn_mask_lid.cnv; + ggml_tensor * indexer_kq_mask = inp_attn->get_dsa()->self_attn_mask_lid.raw; indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask); cb(indexer_score, "indexer_score", il); } diff --git a/src/models/glm-dsa.cpp b/src/models/glm-dsa.cpp index 0bf7a7c85..146112d36 100644 --- a/src/models/glm-dsa.cpp +++ b/src/models/glm-dsa.cpp @@ -305,7 +305,7 @@ llama_model_glm_dsa::graph::graph(const llama_model & model, const llm_graph_par ggml_tensor * indexer_score = nullptr; if (cparams.fused_lid) { - indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.cnv); + indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.raw); cb(indexer_score, "indexer_score", il); res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il}); } else { @@ -339,7 +339,7 @@ llama_model_glm_dsa::graph::graph(const llama_model & model, const llm_graph_par cb(indexer_score, "indexer_score", il); // mask indexer scores - ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.cnv; + ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.raw; indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask); cb(indexer_score, "indexer_score", il); } diff --git a/src/models/hy-v4.cpp b/src/models/hy-v4.cpp index 06833e20b..945e0cc37 100644 --- a/src/models/hy-v4.cpp +++ b/src/models/hy-v4.cpp @@ -395,7 +395,7 @@ ggml_tensor * llama_model_hy_v4::graph::build_indexer_top_k( ggml_tensor * score = nullptr; if (cparams.fused_lid) { - score = ggml_lightning_indexer(ctx0, iq, ik, iw, inp_attn_dsa->self_attn_mask_lid.cnv); + score = ggml_lightning_indexer(ctx0, iq, ik, iw, inp_attn_dsa->self_attn_mask_lid.raw); cb(score, "indexer_score", il); res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, score, il}); } else { @@ -408,7 +408,7 @@ ggml_tensor * llama_model_hy_v4::graph::build_indexer_top_k( score = ggml_mul(ctx0, score, iw); score = ggml_sum_rows(ctx0, score); score = ggml_cont(ctx0, ggml_permute(ctx0, score, 2, 1, 0, 3)); - score = ggml_add(ctx0, score, inp_attn_dsa->self_attn_mask_lid.cnv); + score = ggml_add(ctx0, score, inp_attn_dsa->self_attn_mask_lid.raw); cb(score, "indexer_score", il); } diff --git a/src/models/lfm2.cpp b/src/models/lfm2.cpp index 3488b6c77..e18937faf 100644 --- a/src/models/lfm2.cpp +++ b/src/models/lfm2.cpp @@ -332,7 +332,6 @@ llama_model_lfm2::graph_decision::graph_decision(const llama_model & model, cons auto inp = std::make_unique(hparams, cparams, is_head); inp->self_attn_mask.raw = ggml_new_tensor_4d(ctx0, type_mask, n_tokens, n_tokens, 1, 1); ggml_set_input(inp->self_attn_mask.raw); - inp->self_attn_mask.cnv = inp->self_attn_mask.raw; inp_attn[is_head] = (llm_graph_input_attn_no_cache *) res->add_input(std::move(inp)); } diff --git a/src/models/minimax-m3.cpp b/src/models/minimax-m3.cpp index 0da9d5a44..6d1528096 100644 --- a/src/models/minimax-m3.cpp +++ b/src/models/minimax-m3.cpp @@ -262,7 +262,7 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ if (msa_enabled) { const auto * mctx_msa = static_cast(mctx); - msa_kqm = inp_attn->self_attn_mask.cnv; + msa_kqm = inp_attn->self_attn_mask.raw; n_kv = msa_kqm->ne[0]; n_tps = msa_kqm->ne[1]; // tokens per stream ns = msa_kqm->ne[3]; // streams in this ubatch diff --git a/src/models/mistral-large4.cpp b/src/models/mistral-large4.cpp index dbd1907a4..a196e6d5b 100644 --- a/src/models/mistral-large4.cpp +++ b/src/models/mistral-large4.cpp @@ -289,7 +289,7 @@ llama_model_mistral_large4::graph::graph(const llama_model & model, const llm_gr ggml_tensor * indexer_score = nullptr; if (cparams.fused_lid) { - indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.cnv); + indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.raw); cb(indexer_score, "indexer_score", il); res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il}); } else { @@ -323,7 +323,7 @@ llama_model_mistral_large4::graph::graph(const llama_model & model, const llm_gr cb(indexer_score, "indexer_score", il); // mask indexer scores - ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.cnv; + ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.raw; indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask); cb(indexer_score, "indexer_score", il); }