llama: create new fused op for QSA mask (#30640)
* llama: create new fused op for QSA scoring * enroll glm5.3-next too * tests : initialize the remaining tensors in test_kpool_mask The harness sentinels were left uninitialized, so leftover memory could read as NaN and fail the NaN check. * ggml: generalize the k-pool mask op into ggml_top_k_mask The k-pool mask op only covered the pooled QSA selection of qwen4exp and glm5-next, while the DSA models and the DeepSeek V4 CSA build the same top-k mask by hand with fill, set_rows and add. ggml_top_k_mask(mask, top_k, score, blocks, extra) keeps the mask cells of the live picks and the extra cells, and sets the rest to -inf. Every row tensor is 4D in the layout of top_k, and the mask is broadcast to those rows, so a selection can also be made per head. score, blocks and extra are optional: without score every pick is live, without blocks a pick is a cell, and extra holds the cells a row keeps without a pick. llm_graph_context::build_fused_top_k_mask views the row tensors in the layout of kq_mask and registers the fused node. The k-pool models, the DSA attention of build_attn and the DeepSeek V4 CSA mask use it when the probe enables the op, and keep their current path on backends without it. --------- Co-authored-by: Pascal <admin@serveurperso.com>
This commit is contained in:
1 parent
0756e83fc1
commit
d454a204ef
18 files changed
+592
-31
No files matched your search
@@ -165,6 +165,12 @@ static const llm_fused_op_probe llm_fused_op_dsv4_hc_post_probe = {
|
||||
/*.n_tokens_per_seq =*/ 1,
|
||||
};
|
||||
|
||||
static const llm_fused_op_probe llm_fused_op_top_k_mask_probe = {
|
||||
/*.op =*/ LLM_FUSED_OP_TOP_K_MASK,
|
||||
/*.name =*/ "fused top-k mask",
|
||||
/*.n_tokens_per_seq =*/ 1,
|
||||
};
|
||||
|
||||
llama_context::llama_context(
|
||||
const llama_model & model,
|
||||
llama_context_params params) :
|
||||
@@ -328,6 +334,10 @@ llama_context::llama_context(
|
||||
cparams.fused_dsv4_hc_post = true;
|
||||
cparams.auto_fhc = true;
|
||||
|
||||
// only the indexer models build this mask, skip the probe for the others
|
||||
cparams.fused_top_k_mask = true;
|
||||
cparams.auto_ftkm = hparams.indexer_top_k > 0;
|
||||
|
||||
// with causal attention, the batch size is limited by the context size
|
||||
cparams.n_batch = cparams.causal_attn ? std::min(cparams.n_ctx, params.n_batch) : params.n_batch;
|
||||
|
||||
@@ -671,6 +681,12 @@ void llama_context::resolve_fused_ops(const llama_memory_context_i * mctx, uint3
|
||||
resolve(llm_fused_op_dsv4_hc_post_probe, cparams.fused_dsv4_hc_post);
|
||||
cparams.auto_fhc = false;
|
||||
}
|
||||
|
||||
if (cparams.auto_ftkm) {
|
||||
LLAMA_LOG_INFO("%s: resolving fused top-k mask support:\n", func);
|
||||
resolve(llm_fused_op_top_k_mask_probe, cparams.fused_top_k_mask);
|
||||
cparams.auto_ftkm = false;
|
||||
}
|
||||
}
|
||||
|
||||
static int llama_graph_n_input_tensors(ggml_cgraph * gf) {
|
||||
|
||||
Reference in new issue
Block a user