llama: create new fused op for QSA mask (#30640)
* llama: create new fused op for QSA scoring * enroll glm5.3-next too * tests : initialize the remaining tensors in test_kpool_mask The harness sentinels were left uninitialized, so leftover memory could read as NaN and fail the NaN check. * ggml: generalize the k-pool mask op into ggml_top_k_mask The k-pool mask op only covered the pooled QSA selection of qwen4exp and glm5-next, while the DSA models and the DeepSeek V4 CSA build the same top-k mask by hand with fill, set_rows and add. ggml_top_k_mask(mask, top_k, score, blocks, extra) keeps the mask cells of the live picks and the extra cells, and sets the rest to -inf. Every row tensor is 4D in the layout of top_k, and the mask is broadcast to those rows, so a selection can also be made per head. score, blocks and extra are optional: without score every pick is live, without blocks a pick is a cell, and extra holds the cells a row keeps without a pick. llm_graph_context::build_fused_top_k_mask views the row tensors in the layout of kq_mask and registers the fused node. The k-pool models, the DSA attention of build_attn and the DeepSeek V4 CSA mask use it when the probe enables the op, and keep their current path on backends without it. --------- Co-authored-by: Pascal <admin@serveurperso.com>
This commit is contained in:
1 parent
0756e83fc1
commit
d454a204ef
18 files changed
+592
-31
No files matched your search
+45
-19
@@ -3375,6 +3375,27 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
return cur;
|
||||
}
|
||||
|
||||
ggml_tensor * llm_graph_context::build_fused_top_k_mask(
|
||||
ggml_tensor * kq_mask,
|
||||
ggml_tensor * top_k,
|
||||
ggml_tensor * score,
|
||||
ggml_tensor * blocks,
|
||||
ggml_tensor * extra,
|
||||
int il) const {
|
||||
const int64_t m1 = kq_mask->ne[1];
|
||||
const int64_t m2 = kq_mask->ne[2];
|
||||
const int64_t m3 = kq_mask->ne[3];
|
||||
|
||||
auto rows = [&](ggml_tensor * t) {
|
||||
return t ? ggml_reshape_4d(ctx0, t, t->ne[0], m1, m2, m3) : nullptr;
|
||||
};
|
||||
|
||||
ggml_tensor * cur = ggml_top_k_mask(ctx0, kq_mask, rows(top_k), rows(score), blocks, rows(extra));
|
||||
res->add_fused_node({LLM_FUSED_OP_TOP_K_MASK, cur, il});
|
||||
|
||||
return cur;
|
||||
}
|
||||
|
||||
ggml_tensor * llm_graph_context::build_attn(
|
||||
llm_graph_input_attn_k_dsa * inp,
|
||||
ggml_tensor * wo,
|
||||
@@ -3407,31 +3428,36 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
|
||||
const auto & kq_mask = inp->get_kq_mask_mla();
|
||||
|
||||
// prepare new kq mask - starts filled with -INFINITY
|
||||
ggml_tensor * kq_mask_all = ggml_fill(ctx0, kq_mask, -INFINITY);
|
||||
ggml_tensor * kq_mask_top_k = nullptr;
|
||||
if (cparams.fused_top_k_mask) {
|
||||
kq_mask_top_k = build_fused_top_k_mask(kq_mask, top_k, nullptr, nullptr, nullptr, il);
|
||||
} else {
|
||||
// prepare new kq mask - starts filled with -INFINITY
|
||||
ggml_tensor * kq_mask_all = ggml_fill(ctx0, kq_mask, -INFINITY);
|
||||
|
||||
// reshape KQ mask into tensor with rows of size 1:
|
||||
// [n_kv, n_batch, 1, n_stream] -> [1, n_kv, n_batch, n_stream]
|
||||
kq_mask_all = ggml_view_4d(ctx0, kq_mask_all, 1, kq_mask_all->ne[0], kq_mask_all->ne[1], kq_mask_all->ne[3], kq_mask_all->nb[0], kq_mask_all->nb[1], kq_mask_all->nb[2], 0);
|
||||
// reshape KQ mask into tensor with rows of size 1:
|
||||
// [n_kv, n_batch, 1, n_stream] -> [1, n_kv, n_batch, n_stream]
|
||||
kq_mask_all = ggml_view_4d(ctx0, kq_mask_all, 1, kq_mask_all->ne[0], kq_mask_all->ne[1], kq_mask_all->ne[3], kq_mask_all->nb[0], kq_mask_all->nb[1], kq_mask_all->nb[2], 0);
|
||||
|
||||
// reshape top_k indices: [n_top_k, n_batch, 1, n_stream] -> [n_top_k, n_batch, n_stream, 1]
|
||||
ggml_tensor * top_k_3d = ggml_view_4d(ctx0, top_k, top_k->ne[0], top_k->ne[1], top_k->ne[3], 1, top_k->nb[1], top_k->nb[2], top_k->ne[3]*top_k->nb[3], 0);
|
||||
// reshape top_k indices: [n_top_k, n_batch, 1, n_stream] -> [n_top_k, n_batch, n_stream, 1]
|
||||
ggml_tensor * top_k_3d = ggml_view_4d(ctx0, top_k, top_k->ne[0], top_k->ne[1], top_k->ne[3], 1, top_k->nb[1], top_k->nb[2], top_k->ne[3]*top_k->nb[3], 0);
|
||||
|
||||
// prepare zero-filled tensor with rows of size 1: [1, n_top_k, n_batch, n_stream]
|
||||
// this will be our source of zero values for unmasking top k mask elements
|
||||
ggml_tensor * zeros = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, 1, top_k_3d->ne[0], top_k_3d->ne[1], top_k_3d->ne[2]);
|
||||
zeros = ggml_fill(ctx0, zeros, 0.0f);
|
||||
// prepare zero-filled tensor with rows of size 1: [1, n_top_k, n_batch, n_stream]
|
||||
// this will be our source of zero values for unmasking top k mask elements
|
||||
ggml_tensor * zeros = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, 1, top_k_3d->ne[0], top_k_3d->ne[1], top_k_3d->ne[2]);
|
||||
zeros = ggml_fill(ctx0, zeros, 0.0f);
|
||||
|
||||
// modify KQ mask by unmasking elements that are in top_k indices
|
||||
// ggml_set_rows([1, n_kv, n_batch, n_stream], [1, n_top_k, n_batch, n_stream], [n_top_k, n_batch, n_stream, 1])
|
||||
ggml_tensor * kq_mask_top_k = ggml_set_rows(ctx0, kq_mask_all, zeros, top_k_3d);
|
||||
// modify KQ mask by unmasking elements that are in top_k indices
|
||||
// ggml_set_rows([1, n_kv, n_batch, n_stream], [1, n_top_k, n_batch, n_stream], [n_top_k, n_batch, n_stream, 1])
|
||||
kq_mask_top_k = ggml_set_rows(ctx0, kq_mask_all, zeros, top_k_3d);
|
||||
|
||||
// reshape to restore the original shape of KQ mask:
|
||||
// [1, n_kv, n_batch, n_stream] -> [n_kv, n_batch, 1, n_stream]
|
||||
kq_mask_top_k = ggml_view_4d(ctx0, kq_mask_top_k, kq_mask_top_k->ne[1], kq_mask_top_k->ne[2], 1, kq_mask_top_k->ne[3], kq_mask_top_k->nb[2], kq_mask_top_k->nb[3], kq_mask_top_k->nb[3], 0);
|
||||
// reshape to restore the original shape of KQ mask:
|
||||
// [1, n_kv, n_batch, n_stream] -> [n_kv, n_batch, 1, n_stream]
|
||||
kq_mask_top_k = ggml_view_4d(ctx0, kq_mask_top_k, kq_mask_top_k->ne[1], kq_mask_top_k->ne[2], 1, kq_mask_top_k->ne[3], kq_mask_top_k->nb[2], kq_mask_top_k->nb[3], kq_mask_top_k->nb[3], 0);
|
||||
|
||||
// combine with the original kq mask
|
||||
kq_mask_top_k = ggml_add(ctx0, kq_mask_top_k, kq_mask);
|
||||
// combine with the original kq mask
|
||||
kq_mask_top_k = ggml_add(ctx0, kq_mask_top_k, kq_mask);
|
||||
}
|
||||
|
||||
ggml_tensor * q = q_cur;
|
||||
ggml_tensor * k = mctx_cur->get_k(ctx0, il);
|
||||
|
||||
Reference in new issue
Block a user