llama: create new fused op for QSA mask (#30640)
* llama: create new fused op for QSA scoring * enroll glm5.3-next too * tests : initialize the remaining tensors in test_kpool_mask The harness sentinels were left uninitialized, so leftover memory could read as NaN and fail the NaN check. * ggml: generalize the k-pool mask op into ggml_top_k_mask The k-pool mask op only covered the pooled QSA selection of qwen4exp and glm5-next, while the DSA models and the DeepSeek V4 CSA build the same top-k mask by hand with fill, set_rows and add. ggml_top_k_mask(mask, top_k, score, blocks, extra) keeps the mask cells of the live picks and the extra cells, and sets the rest to -inf. Every row tensor is 4D in the layout of top_k, and the mask is broadcast to those rows, so a selection can also be made per head. score, blocks and extra are optional: without score every pick is live, without blocks a pick is a cell, and extra holds the cells a row keeps without a pick. llm_graph_context::build_fused_top_k_mask views the row tensors in the layout of kq_mask and registers the fused node. The k-pool models, the DSA attention of build_attn and the DeepSeek V4 CSA mask use it when the probe enables the op, and keep their current path on backends without it. --------- Co-authored-by: Pascal <admin@serveurperso.com>
This commit is contained in:
1 parent
0756e83fc1
commit
d454a204ef
18 files changed
+592
-31
No files matched your search
@@ -323,13 +323,15 @@ llama_model_glm5_next::llm_graph_input_kpool * llama_model_glm5_next::graph::bui
|
||||
const int64_t n_sel = kpool*n_top_pool + (hparams.indexer_kpool_select_tail ? kpool - 1 : 0);
|
||||
inp->n_sel = (uint32_t) n_sel;
|
||||
|
||||
// the scatter maps the dead slots of this mask to dump rows
|
||||
inp->sel_mask = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, n_sel, 1, 1, n_tokens);
|
||||
ggml_set_input(inp->sel_mask);
|
||||
// set_input_kpool always fills the mask, so it stays allocated in every graph
|
||||
ggml_build_forward_expand(gf, inp->sel_mask);
|
||||
// one view for all layers: each view of a host input is a separate upload
|
||||
inp->sel_mask_2d = ggml_reshape_2d(ctx0, inp->sel_mask, n_sel, n_tokens);
|
||||
// the scatter maps the dead slots of this mask to dump rows, the fused mask op reads liveness from the scores instead
|
||||
if (!cparams.fused_top_k_mask) {
|
||||
inp->sel_mask = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, n_sel, 1, 1, n_tokens);
|
||||
ggml_set_input(inp->sel_mask);
|
||||
// set_input_kpool always fills the mask, so it stays allocated in every graph
|
||||
ggml_build_forward_expand(gf, inp->sel_mask);
|
||||
// one view for all layers: each view of a host input is a separate upload
|
||||
inp->sel_mask_2d = ggml_reshape_2d(ctx0, inp->sel_mask, n_sel, n_tokens);
|
||||
}
|
||||
}
|
||||
|
||||
inp->n_new = n_new;
|
||||
@@ -973,6 +975,17 @@ ggml_tensor * llama_model_glm5_next::graph::build_kpool_select(
|
||||
const int64_t n_top_pool = std::min<int64_t>(n_pool, hparams.indexer_top_k / kpool);
|
||||
ggml_tensor * top_k = ggml_top_k(ctx0, score, n_top_pool); // [n_top_pool, n_tokens], UNORDERED
|
||||
|
||||
// keep the kq_mask cells of the live picked pools and the tail, -inf elsewhere
|
||||
if (cparams.fused_top_k_mask) {
|
||||
ggml_tensor * tail = hparams.indexer_kpool_select_tail ? inp_kpool->tail_idxs : nullptr;
|
||||
GGML_ASSERT(kpool*n_top_pool + (tail ? tail->ne[0] : 0) == (int64_t) inp_kpool->n_sel);
|
||||
|
||||
ggml_tensor * sel = build_fused_top_k_mask(kq_mask, top_k, score, inp_kpool->pool_idxs, tail, il);
|
||||
cb(sel, "indexer_sel", il);
|
||||
|
||||
return sel;
|
||||
}
|
||||
|
||||
// The selection mask marks the first min(nv, n_top_pool) slots as the visible pools, so order the set by descending score.
|
||||
ggml_tensor * sel_score = ggml_get_rows(ctx0,
|
||||
ggml_reshape_3d(ctx0, score, 1, n_pool, n_tokens), top_k); // [1, n_top_pool, n_tokens]
|
||||
|
||||
Reference in new issue
Block a user