llama : build the k-pool sparse selection independently of the kq_mask shape (#31243)
* llama : build the k-pool sparse selection independently of the kq_mask shape qwen4exp (QSA) and glm5-next pick cache cells by index and no longer read the kq_mask values, but they still shaped the selection like the dense kq_mask. Build the selection as [n_kv, n_tokens] over the cache cells instead, and split it into the streams of the K view right before the attention. This lets the dense mask change its layout (e.g. per-sequence row lists) without touching the sparse layers. * improve comment Co-authored-by: Georgi Gerganov <ggerganov@gmail.com> * llama : drop the unused build_fused_top_k_mask args, review nits --------- Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
This commit is contained in:
1 parent
7ecb57f7ad
commit
27b7a8a009
6 files changed
+41
-57
No files matched your search
+3
-13
@@ -3398,20 +3398,10 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
ggml_tensor * llm_graph_context::build_fused_top_k_mask(
|
||||
ggml_tensor * kq_mask,
|
||||
ggml_tensor * top_k,
|
||||
ggml_tensor * score,
|
||||
ggml_tensor * blocks,
|
||||
ggml_tensor * extra,
|
||||
ggml_tensor * mask,
|
||||
int il) const {
|
||||
const int64_t m1 = kq_mask->ne[1];
|
||||
const int64_t m2 = kq_mask->ne[2];
|
||||
const int64_t m3 = kq_mask->ne[3];
|
||||
top_k = ggml_reshape_4d(ctx0, top_k, top_k->ne[0], kq_mask->ne[1], kq_mask->ne[2], kq_mask->ne[3]);
|
||||
|
||||
auto rows = [&](ggml_tensor * t) {
|
||||
return t ? ggml_reshape_4d(ctx0, t, t->ne[0], m1, m2, m3) : nullptr;
|
||||
};
|
||||
|
||||
ggml_tensor * cur = ggml_top_k_mask(ctx0, mask ? mask : kq_mask, rows(top_k), rows(score), blocks, rows(extra));
|
||||
ggml_tensor * cur = ggml_top_k_mask(ctx0, kq_mask, top_k, nullptr, nullptr, nullptr);
|
||||
res->add_fused_node({LLM_FUSED_OP_TOP_K_MASK, cur, il});
|
||||
|
||||
return cur;
|
||||
@@ -3456,7 +3446,7 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
|
||||
ggml_tensor * kq_mask_top_k = nullptr;
|
||||
if (cparams.fused_top_k_mask) {
|
||||
kq_mask_top_k = build_fused_top_k_mask(kq_mask, top_k, nullptr, nullptr, nullptr, nullptr, il);
|
||||
kq_mask_top_k = build_fused_top_k_mask(kq_mask, top_k, il);
|
||||
} else {
|
||||
// prepare new kq mask - starts filled with -INFINITY
|
||||
ggml_tensor * kq_mask_all = ggml_fill(ctx0, kq_mask, -INFINITY);
|
||||
|
||||
Reference in new issue
Block a user