qwen4exp : build the QSA selection without the kq_mask (#30965)
* qwen4exp : build the QSA selection without the kq_mask * clean * lang * qwen4exp : reshape the fused QSA selection to the kq_mask streams ggml_top_k_mask with a single keep row returns [n_kv, n_tokens], while build_attn_qsa expects the kq_mask layout [n_kv, n_tokens/n_stream, 1, n_stream]. With several streams the shape assert failed at graph reserve. * test the QSA optimization * reword * update build_fused_top_k_mask instead of handwiring * simplify * keep il at the end for consistency
This commit is contained in:
1 parent
da2ee1e7a6
commit
5c31142682
6 files changed
+28
-14
No files matched your search
@@ -831,11 +831,13 @@ ggml_tensor * llama_model_qwen4exp::graph::build_qsa_sel(
|
||||
ggml_tensor * top_k = ggml_top_k(ctx0, score, n_top_pool); // [n_top_pool, n_tokens], unordered
|
||||
cb(top_k, "indexer_top_k", il);
|
||||
|
||||
// keep the kq_mask cells of the live picked pools and the tail, -inf elsewhere
|
||||
// keep the cells of the live picked pools and the tail, -inf elsewhere
|
||||
// all retained cells are in the token's sequence, so the kq_mask is not needed
|
||||
if (cparams.fused_top_k_mask) {
|
||||
GGML_ASSERT(kpool*n_top_pool + inp_kpool->tail_idxs->ne[0] == inp_kpool->n_sel);
|
||||
|
||||
ggml_tensor * sel = build_fused_top_k_mask(kq_mask, top_k, score, inp_kpool->pool_idxs, inp_kpool->tail_idxs, il);
|
||||
ggml_tensor * keep = ggml_fill(ctx0, ggml_new_tensor_1d(ctx0, kq_mask->type, inp_kpool->n_kv), 0.0f);
|
||||
ggml_tensor * sel = build_fused_top_k_mask(kq_mask, top_k, score, inp_kpool->pool_idxs, inp_kpool->tail_idxs, keep, il);
|
||||
cb(sel, "indexer_sel", il);
|
||||
|
||||
return sel;
|
||||
@@ -891,7 +893,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_qsa_sel(
|
||||
const size_t row = sel->nb[2];
|
||||
sel = ggml_view_4d(ctx0, sel, n_kv, kq_mask->ne[1], kq_mask->ne[2], kq_mask->ne[3],
|
||||
row, row*kq_mask->ne[1], row*kq_mask->ne[1]*kq_mask->ne[2], 0);
|
||||
sel = ggml_add(ctx0, sel, kq_mask);
|
||||
sel = ggml_cont(ctx0, sel);
|
||||
cb(sel, "indexer_sel", il);
|
||||
|
||||
return sel;
|
||||
@@ -936,8 +938,9 @@ ggml_tensor * llama_model_qwen4exp::graph::build_attn_qsa(
|
||||
ggml_build_forward_expand(gf, mctx_cur->cpy_v(ctx0, v_cur, v_idxs, il));
|
||||
}
|
||||
|
||||
// the selection already carries the causal mask, and it comes in the mask layout
|
||||
ggml_tensor * kq_mask = inp->get_kq_mask();
|
||||
// the selection replaces the kq_mask and has the same shape
|
||||
// it is not allocated without dense layers reading the kq_mask
|
||||
const ggml_tensor * kq_mask = inp->get_kq_mask();
|
||||
GGML_ASSERT(ggml_are_same_shape(sel, kq_mask));
|
||||
cb(sel, "kq_mask_qsa", il);
|
||||
|
||||
|
||||
Reference in new issue
Block a user