kv-cache : keep rotation constants on the GPU (#31073)
* kv-cache : keep rotation constants on the GPU Assisted-by: Codex * tests : verify resident KV rotation buffers Assisted-by: Codex * kv-cache : document remaining multi-device rotation work Assisted-by: Codex * kv-cache : support per-layer rotation buffers Assisted-by: Codex * kv-cache : fix hybrid SWA rotation inputs and mixed placement check Assisted-by: Codex * models : use per-layer rotation in mistral-large4 and dflash * tests: drop the rotation buffer tests --------- Co-authored-by: Pascal <admin@serveurperso.com>
This commit is contained in:
1 parent
ab3768cdc2
commit
fb5997274e
15 files changed
+179
-246
No files matched your search
+12
-12
@@ -980,8 +980,8 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra
|
||||
ggml_tensor * k_idxs = is_swa ? inp_attn_iswa->get_k_idxs_swa() : inp_attn_iswa->get_k_idxs();
|
||||
ggml_tensor * v_idxs = is_swa ? inp_attn_iswa->get_v_idxs_swa() : inp_attn_iswa->get_v_idxs();
|
||||
// rotate K/V into the cache's rotated space
|
||||
ggml_tensor * k_rot = is_swa ? inp_attn_iswa->self_k_rot_swa : inp_attn_iswa->self_k_rot;
|
||||
ggml_tensor * v_rot = is_swa ? inp_attn_iswa->self_v_rot_swa : inp_attn_iswa->self_v_rot;
|
||||
ggml_tensor * k_rot = is_swa ? inp_attn_iswa->self_k_rot_swa[il] : inp_attn_iswa->self_k_rot[il];
|
||||
ggml_tensor * v_rot = is_swa ? inp_attn_iswa->self_v_rot_swa[il] : inp_attn_iswa->self_v_rot[il];
|
||||
if (k_rot) {
|
||||
Kcur = llama_mul_mat_hadamard(ctx0, Kcur, k_rot);
|
||||
}
|
||||
@@ -992,11 +992,11 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra
|
||||
ggml_build_forward_expand(gf, kv->cpy_v(ctx0, Vcur, v_idxs, il));
|
||||
} else {
|
||||
// rotate K/V into the cache's rotated space
|
||||
if (inp_attn->self_k_rot) {
|
||||
Kcur = llama_mul_mat_hadamard(ctx0, Kcur, inp_attn->self_k_rot);
|
||||
if (inp_attn->self_k_rot[il]) {
|
||||
Kcur = llama_mul_mat_hadamard(ctx0, Kcur, inp_attn->self_k_rot[il]);
|
||||
}
|
||||
if (inp_attn->self_v_rot) {
|
||||
Vcur = llama_mul_mat_hadamard(ctx0, Vcur, inp_attn->self_v_rot);
|
||||
if (inp_attn->self_v_rot[il]) {
|
||||
Vcur = llama_mul_mat_hadamard(ctx0, Vcur, inp_attn->self_v_rot[il]);
|
||||
}
|
||||
ggml_build_forward_expand(gf, inp_attn->mctx->cpy_k(ctx0, Kcur, inp_attn->get_k_idxs(), il));
|
||||
ggml_build_forward_expand(gf, inp_attn->mctx->cpy_v(ctx0, Vcur, inp_attn->get_v_idxs(), il));
|
||||
@@ -1249,8 +1249,8 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_
|
||||
kv = ggml_rope_set_offset(kv, n_embd_head_nope);
|
||||
cb(kv, "kv_injected", il);
|
||||
|
||||
if (inp_attn->self_k_rot_swa) {
|
||||
kv = llama_mul_mat_hadamard(ctx0, kv, inp_attn->self_k_rot_swa);
|
||||
if (inp_attn->self_k_rot_swa[il]) {
|
||||
kv = llama_mul_mat_hadamard(ctx0, kv, inp_attn->self_k_rot_swa[il]);
|
||||
}
|
||||
ggml_build_forward_expand(gf, inp_attn->mctx->get_swa()->cpy_k(ctx0, kv, inp_attn->get_k_idxs_swa(), il));
|
||||
}
|
||||
@@ -1441,8 +1441,8 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll
|
||||
kv = rope(kv);
|
||||
cb(kv, "kv_injected", il);
|
||||
|
||||
if (inp_attn->self_k_rot_swa) {
|
||||
kv = llama_mul_mat_hadamard(ctx0, kv, inp_attn->self_k_rot_swa);
|
||||
if (inp_attn->self_k_rot_swa[il]) {
|
||||
kv = llama_mul_mat_hadamard(ctx0, kv, inp_attn->self_k_rot_swa[il]);
|
||||
}
|
||||
ggml_build_forward_expand(gf, inp_attn->mctx->get_swa()->cpy_k(ctx0, kv, inp_attn->get_k_idxs_swa(), il));
|
||||
}
|
||||
@@ -1686,8 +1686,8 @@ llama_model_dflash::graph_mistral_large4::graph_mistral_large4(const llama_model
|
||||
ggml_tensor * kv = build_kv(inp_g, il).first;
|
||||
cb(kv, "kv_injected", il);
|
||||
|
||||
if (inp_attn->self_k_rot_swa) {
|
||||
kv = llama_mul_mat_hadamard(ctx0, kv, inp_attn->self_k_rot_swa);
|
||||
if (inp_attn->self_k_rot_swa[il]) {
|
||||
kv = llama_mul_mat_hadamard(ctx0, kv, inp_attn->self_k_rot_swa[il]);
|
||||
}
|
||||
ggml_build_forward_expand(gf, inp_attn->mctx->get_swa()->cpy_k(ctx0, kv, inp_attn->get_k_idxs_swa(), il));
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user