kv-cache : keep rotation constants on the GPU (#31073)

* kv-cache : keep rotation constants on the GPU

Assisted-by: Codex

* tests : verify resident KV rotation buffers

Assisted-by: Codex

* kv-cache : document remaining multi-device rotation work

Assisted-by: Codex

* kv-cache : support per-layer rotation buffers

Assisted-by: Codex

* kv-cache : fix hybrid SWA rotation inputs and mixed placement check

Assisted-by: Codex

* models : use per-layer rotation in mistral-large4 and dflash

* tests: drop the rotation buffer tests

---------

Co-authored-by: Pascal <admin@serveurperso.com>
This commit is contained in:
Anton IandPascal authored and GitHub committed 2026-10-30 10:15:06 +02:00
1 parent ab3768cdc2
commit fb5997274e
15 files changed
+179 -246

No files matched your search

+12 -12
View File
@@ -980,8 +980,8 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra
ggml_tensor * k_idxs = is_swa ? inp_attn_iswa->get_k_idxs_swa() : inp_attn_iswa->get_k_idxs();
ggml_tensor * v_idxs = is_swa ? inp_attn_iswa->get_v_idxs_swa() : inp_attn_iswa->get_v_idxs();
// rotate K/V into the cache's rotated space
ggml_tensor * k_rot = is_swa ? inp_attn_iswa->self_k_rot_swa : inp_attn_iswa->self_k_rot;
ggml_tensor * v_rot = is_swa ? inp_attn_iswa->self_v_rot_swa : inp_attn_iswa->self_v_rot;
ggml_tensor * k_rot = is_swa ? inp_attn_iswa->self_k_rot_swa[il] : inp_attn_iswa->self_k_rot[il];
ggml_tensor * v_rot = is_swa ? inp_attn_iswa->self_v_rot_swa[il] : inp_attn_iswa->self_v_rot[il];
if (k_rot) {
Kcur = llama_mul_mat_hadamard(ctx0, Kcur, k_rot);
}
@@ -992,11 +992,11 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra
ggml_build_forward_expand(gf, kv->cpy_v(ctx0, Vcur, v_idxs, il));
} else {
// rotate K/V into the cache's rotated space
if (inp_attn->self_k_rot) {
Kcur = llama_mul_mat_hadamard(ctx0, Kcur, inp_attn->self_k_rot);
if (inp_attn->self_k_rot[il]) {
Kcur = llama_mul_mat_hadamard(ctx0, Kcur, inp_attn->self_k_rot[il]);
}
if (inp_attn->self_v_rot) {
Vcur = llama_mul_mat_hadamard(ctx0, Vcur, inp_attn->self_v_rot);
if (inp_attn->self_v_rot[il]) {
Vcur = llama_mul_mat_hadamard(ctx0, Vcur, inp_attn->self_v_rot[il]);
}
ggml_build_forward_expand(gf, inp_attn->mctx->cpy_k(ctx0, Kcur, inp_attn->get_k_idxs(), il));
ggml_build_forward_expand(gf, inp_attn->mctx->cpy_v(ctx0, Vcur, inp_attn->get_v_idxs(), il));
@@ -1249,8 +1249,8 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_
kv = ggml_rope_set_offset(kv, n_embd_head_nope);
cb(kv, "kv_injected", il);
if (inp_attn->self_k_rot_swa) {
kv = llama_mul_mat_hadamard(ctx0, kv, inp_attn->self_k_rot_swa);
if (inp_attn->self_k_rot_swa[il]) {
kv = llama_mul_mat_hadamard(ctx0, kv, inp_attn->self_k_rot_swa[il]);
}
ggml_build_forward_expand(gf, inp_attn->mctx->get_swa()->cpy_k(ctx0, kv, inp_attn->get_k_idxs_swa(), il));
}
@@ -1441,8 +1441,8 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll
kv = rope(kv);
cb(kv, "kv_injected", il);
if (inp_attn->self_k_rot_swa) {
kv = llama_mul_mat_hadamard(ctx0, kv, inp_attn->self_k_rot_swa);
if (inp_attn->self_k_rot_swa[il]) {
kv = llama_mul_mat_hadamard(ctx0, kv, inp_attn->self_k_rot_swa[il]);
}
ggml_build_forward_expand(gf, inp_attn->mctx->get_swa()->cpy_k(ctx0, kv, inp_attn->get_k_idxs_swa(), il));
}
@@ -1686,8 +1686,8 @@ llama_model_dflash::graph_mistral_large4::graph_mistral_large4(const llama_model
ggml_tensor * kv = build_kv(inp_g, il).first;
cb(kv, "kv_injected", il);
if (inp_attn->self_k_rot_swa) {
kv = llama_mul_mat_hadamard(ctx0, kv, inp_attn->self_k_rot_swa);
if (inp_attn->self_k_rot_swa[il]) {
kv = llama_mul_mat_hadamard(ctx0, kv, inp_attn->self_k_rot_swa[il]);
}
ggml_build_forward_expand(gf, inp_attn->mctx->get_swa()->cpy_k(ctx0, kv, inp_attn->get_k_idxs_swa(), il));
}