qwen4exp : build the QSA selection without the kq_mask (#30965)

* qwen4exp : build the QSA selection without the kq_mask

* clean

* lang

* qwen4exp : reshape the fused QSA selection to the kq_mask streams

ggml_top_k_mask with a single keep row returns [n_kv, n_tokens], while
build_attn_qsa expects the kq_mask layout [n_kv, n_tokens/n_stream, 1, n_stream].
With several streams the shape assert failed at graph reserve.

* test the QSA optimization

* reword

* update build_fused_top_k_mask instead of handwiring

* simplify

* keep il at the end for consistency
This commit is contained in:
Alexey Kopytko authored and GitHub committed 2026-10-23 19:49:08 +02:00
1 parent da2ee1e7a6
commit 5c31142682
6 files changed
+28 -14

No files matched your search

+8 -5
View File
@@ -831,11 +831,13 @@ ggml_tensor * llama_model_qwen4exp::graph::build_qsa_sel(
ggml_tensor * top_k = ggml_top_k(ctx0, score, n_top_pool); // [n_top_pool, n_tokens], unordered
cb(top_k, "indexer_top_k", il);
// keep the kq_mask cells of the live picked pools and the tail, -inf elsewhere
// keep the cells of the live picked pools and the tail, -inf elsewhere
// all retained cells are in the token's sequence, so the kq_mask is not needed
if (cparams.fused_top_k_mask) {
GGML_ASSERT(kpool*n_top_pool + inp_kpool->tail_idxs->ne[0] == inp_kpool->n_sel);
ggml_tensor * sel = build_fused_top_k_mask(kq_mask, top_k, score, inp_kpool->pool_idxs, inp_kpool->tail_idxs, il);
ggml_tensor * keep = ggml_fill(ctx0, ggml_new_tensor_1d(ctx0, kq_mask->type, inp_kpool->n_kv), 0.0f);
ggml_tensor * sel = build_fused_top_k_mask(kq_mask, top_k, score, inp_kpool->pool_idxs, inp_kpool->tail_idxs, keep, il);
cb(sel, "indexer_sel", il);
return sel;
@@ -891,7 +893,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_qsa_sel(
const size_t row = sel->nb[2];
sel = ggml_view_4d(ctx0, sel, n_kv, kq_mask->ne[1], kq_mask->ne[2], kq_mask->ne[3],
row, row*kq_mask->ne[1], row*kq_mask->ne[1]*kq_mask->ne[2], 0);
sel = ggml_add(ctx0, sel, kq_mask);
sel = ggml_cont(ctx0, sel);
cb(sel, "indexer_sel", il);
return sel;
@@ -936,8 +938,9 @@ ggml_tensor * llama_model_qwen4exp::graph::build_attn_qsa(
ggml_build_forward_expand(gf, mctx_cur->cpy_v(ctx0, v_cur, v_idxs, il));
}
// the selection already carries the causal mask, and it comes in the mask layout
ggml_tensor * kq_mask = inp->get_kq_mask();
// the selection replaces the kq_mask and has the same shape
// it is not allocated without dense layers reading the kq_mask
const ggml_tensor * kq_mask = inp->get_kq_mask();
GGML_ASSERT(ggml_are_same_shape(sel, kq_mask));
cb(sel, "kq_mask_qsa", il);