kv-cache : keep rotation constants on the GPU (#31073)

* kv-cache : keep rotation constants on the GPU

Assisted-by: Codex

* tests : verify resident KV rotation buffers

Assisted-by: Codex

* kv-cache : document remaining multi-device rotation work

Assisted-by: Codex

* kv-cache : support per-layer rotation buffers

Assisted-by: Codex

* kv-cache : fix hybrid SWA rotation inputs and mixed placement check

Assisted-by: Codex

* models : use per-layer rotation in mistral-large4 and dflash

* tests: drop the rotation buffer tests

---------

Co-authored-by: Pascal <admin@serveurperso.com>
This commit is contained in:
Anton IandPascal authored and GitHub committed 2026-10-30 10:15:06 +02:00
1 parent ab3768cdc2
commit fb5997274e
15 files changed
+179 -246

No files matched your search

+31 -102
View File
@@ -554,14 +554,6 @@ void llm_graph_input_attn_kv::set_input(const llama_ubatch * ubatch) {
if (self_kq_mask && self_kq_mask->buffer) {
mctx->set_input_kq_mask(self_kq_mask, ubatch, cparams.causal_attn);
}
if (self_k_rot && self_k_rot->buffer) {
mctx->set_input_k_rot(self_k_rot);
}
if (self_v_rot && self_v_rot->buffer) {
mctx->set_input_v_rot(self_v_rot);
}
}
bool llm_graph_input_attn_kv::can_reuse(const llm_graph_params & params) {
@@ -646,11 +638,6 @@ void llm_graph_input_attn_k_dsa::set_input(const llama_ubatch * ubatch) {
mctx->get_lid()->set_input_k_idxs(self_k_idxs_lid, ubatch);
mctx->get_lid()->set_input_kq_mask(self_kq_mask_lid, ubatch, cparams.causal_attn);
// left unallocated when the indexer does not use the rotation
if (self_k_rot_lid && self_k_rot_lid->buffer) {
mctx->get_lid()->set_input_k_rot(self_k_rot_lid);
}
}
bool llm_graph_input_attn_k_dsa::can_reuse(const llm_graph_params & params) {
@@ -715,22 +702,6 @@ void llm_graph_input_attn_kv_iswa::set_input(const llama_ubatch * ubatch) {
if (self_kq_mask_swa && self_kq_mask_swa->buffer) {
mctx->get_swa()->set_input_kq_mask(self_kq_mask_swa, ubatch, cparams.causal_attn);
}
if (self_k_rot && self_k_rot->buffer) {
mctx->get_base()->set_input_k_rot(self_k_rot);
}
if (self_v_rot && self_v_rot->buffer) {
mctx->get_base()->set_input_v_rot(self_v_rot);
}
if (self_k_rot_swa && self_k_rot_swa->buffer) {
mctx->get_swa()->set_input_k_rot(self_k_rot_swa);
}
if (self_v_rot_swa && self_v_rot_swa->buffer) {
mctx->get_swa()->set_input_v_rot(self_v_rot_swa);
}
}
bool llm_graph_input_attn_kv_iswa::can_reuse(const llm_graph_params & params) {
@@ -782,14 +753,6 @@ void llm_graph_input_attn_k_iswa::set_input(const llama_ubatch * ubatch) {
if (self_kq_mask_swa && self_kq_mask_swa->buffer) {
mctx->get_swa()->set_input_kq_mask(self_kq_mask_swa, ubatch, cparams.causal_attn);
}
if (self_k_rot && self_k_rot->buffer) {
mctx->get_base()->set_input_k_rot(self_k_rot);
}
if (self_k_rot_swa && self_k_rot_swa->buffer) {
mctx->get_swa()->set_input_k_rot(self_k_rot_swa);
}
}
bool llm_graph_input_attn_k_iswa::can_reuse(const llm_graph_params & params) {
@@ -1069,10 +1032,6 @@ void llm_graph_input_dsv4_raw::set_input(const llama_ubatch * ubatch) {
if (self_kq_mask && self_kq_mask->buffer) {
mctx->set_input_kq_mask(self_kq_mask, ubatch, cparams.causal_attn);
}
if (self_k_rot) {
mctx->set_input_k_rot(self_k_rot);
}
}
void llm_graph_input_dsv4::set_input(const llama_ubatch * ubatch) {
@@ -1087,18 +1046,6 @@ void llm_graph_input_dsv4::set_input(const llama_ubatch * ubatch) {
dsv4_set_comp_inputs(inp_csa, plan_csa, "csa", debug > 0, ubatch->n_tokens, n_stream);
dsv4_set_comp_inputs(inp_hca, plan_hca, "hca", debug > 0, ubatch->n_tokens, n_stream);
dsv4_set_comp_inputs(inp_lid, plan_lid, "lid", debug > 0, ubatch->n_tokens, n_stream);
if (inp_csa.k_rot && inp_csa.k_rot->buffer) {
mctx->get_csa()->set_input_k_rot(inp_csa.k_rot);
}
if (inp_hca.k_rot && inp_hca.k_rot->buffer) {
mctx->get_hca()->set_input_k_rot(inp_hca.k_rot);
}
if (inp_lid.k_rot && inp_lid.k_rot->buffer) {
mctx->get_lid()->set_input_k_rot(inp_lid.k_rot);
}
}
bool llm_graph_input_dsv4::can_reuse(const llm_graph_params & params) {
@@ -1176,14 +1123,6 @@ void llm_graph_input_mem_hybrid::set_input(const llama_ubatch * ubatch) {
mctx->get_attn()->set_input_kq_mask(inp_attn->self_kq_mask, ubatch, cparams.causal_attn);
}
if (inp_attn->self_k_rot) {
mctx->get_attn()->set_input_k_rot(inp_attn->self_k_rot);
}
if (inp_attn->self_v_rot) {
mctx->get_attn()->set_input_v_rot(inp_attn->self_v_rot);
}
const int64_t n_rs = mctx->get_recr()->get_n_rs();
if (inp_rs->s_copy) {
@@ -1287,22 +1226,6 @@ void llm_graph_input_mem_hybrid_iswa::set_input(const llama_ubatch * ubatch) {
attn_ctx->get_swa()->set_input_kq_mask(inp_attn->self_kq_mask_swa, ubatch, cparams.causal_attn);
}
if (inp_attn->self_k_rot) {
attn_ctx->get_base()->set_input_k_rot(inp_attn->self_k_rot);
}
if (inp_attn->self_v_rot) {
attn_ctx->get_base()->set_input_v_rot(inp_attn->self_v_rot);
}
if (inp_attn->self_k_rot_swa) {
attn_ctx->get_swa()->set_input_k_rot(inp_attn->self_k_rot_swa);
}
if (inp_attn->self_v_rot_swa) {
attn_ctx->get_swa()->set_input_v_rot(inp_attn->self_v_rot_swa);
}
const int64_t n_rs = mctx->get_recr()->get_n_rs();
if (inp_rs->s_copy) {
@@ -3218,8 +3141,8 @@ static std::unique_ptr<llm_graph_input_attn_kv> build_attn_inp_kv_impl(
inp->self_kq_mask_cnv = inp->self_kq_mask;
}
inp->self_k_rot = mctx_cur->build_input_k_rot(ctx0);
inp->self_v_rot = mctx_cur->build_input_v_rot(ctx0);
inp->self_k_rot = mctx_cur->get_k_rot();
inp->self_v_rot = mctx_cur->get_v_rot();
return inp;
}
@@ -3247,13 +3170,13 @@ ggml_tensor * llm_graph_context::build_attn(
int il) const {
GGML_ASSERT(v_mla == nullptr);
if (inp->self_k_rot) {
q_cur = llama_mul_mat_hadamard(ctx0, q_cur, inp->self_k_rot);
k_cur = llama_mul_mat_hadamard(ctx0, k_cur, inp->self_k_rot);
if (inp->self_k_rot[il]) {
q_cur = llama_mul_mat_hadamard(ctx0, q_cur, inp->self_k_rot[il]);
k_cur = llama_mul_mat_hadamard(ctx0, k_cur, inp->self_k_rot[il]);
}
if (inp->self_v_rot) {
v_cur = llama_mul_mat_hadamard(ctx0, v_cur, inp->self_v_rot);
if (inp->self_v_rot[il]) {
v_cur = llama_mul_mat_hadamard(ctx0, v_cur, inp->self_v_rot[il]);
}
// these nodes are added to the graph together so that they are not reordered
@@ -3292,8 +3215,8 @@ ggml_tensor * llm_graph_context::build_attn(
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, 0, kq_scale, il);
cb(cur, "kqv_out", il);
if (inp->self_v_rot) {
cur = llama_mul_mat_hadamard(ctx0, cur, inp->self_v_rot);
if (inp->self_v_rot[il]) {
cur = llama_mul_mat_hadamard(ctx0, cur, inp->self_v_rot[il]);
}
if (wo) {
@@ -3526,8 +3449,8 @@ ggml_tensor * llm_graph_context::build_attn(
int il) const {
const bool is_swa = hparams.is_swa(il);
auto * k_rot = is_swa ? inp->self_k_rot_swa : inp->self_k_rot;
auto * v_rot = is_swa ? inp->self_v_rot_swa : inp->self_v_rot;
auto * k_rot = is_swa ? inp->self_k_rot_swa[il] : inp->self_k_rot[il];
auto * v_rot = is_swa ? inp->self_v_rot_swa[il] : inp->self_v_rot[il];
if (k_rot) {
q_cur = llama_mul_mat_hadamard(ctx0, q_cur, k_rot);
@@ -3621,7 +3544,7 @@ ggml_tensor * llm_graph_context::build_attn(
int il) const {
const bool is_swa = hparams.is_swa(il);
auto * k_rot = is_swa ? inp->self_k_rot_swa : inp->self_k_rot;
auto * k_rot = is_swa ? inp->self_k_rot_swa[il] : inp->self_k_rot[il];
if (k_rot) {
q_cur = llama_mul_mat_hadamard(ctx0, q_cur, k_rot);
@@ -3758,7 +3681,7 @@ static std::unique_ptr<llm_graph_input_attn_k_dsa> build_attn_inp_k_dsa_impl(
inp->self_kq_mask_lid = build_attn_inp_kq_mask(ctx0, mctx_cur->get_lid(), ubatch, cparams_copy);
inp->self_kq_mask_lid_cnv = inp->self_kq_mask_lid;
inp->self_k_rot_lid = mctx_cur->get_lid()->build_input_k_rot(ctx0);
inp->self_k_rot_lid = mctx_cur->get_lid()->get_k_rot();
}
return inp;
@@ -3808,8 +3731,8 @@ llm_graph_input_attn_kv_msa * llm_graph_context::build_attn_inp_kv_msa(bool msa_
inp->self_kq_mask_cnv = inp->self_kq_mask;
}
inp->self_k_rot = mctx_base->build_input_k_rot(ctx0);
inp->self_v_rot = mctx_base->build_input_v_rot(ctx0);
inp->self_k_rot = mctx_base->get_k_rot();
inp->self_v_rot = mctx_base->get_v_rot();
if (msa_enabled) {
inp->self_k_idxs_idx = mctx_idx->build_input_k_idxs(ctx0, ubatch);
@@ -3844,11 +3767,11 @@ llm_graph_input_attn_kv_iswa * llm_graph_context::build_attn_inp_kv_iswa() const
inp->self_kq_mask_swa_cnv = inp->self_kq_mask_swa;
}
inp->self_k_rot = mctx_cur->get_base()->build_input_k_rot(ctx0);
inp->self_v_rot = mctx_cur->get_base()->build_input_v_rot(ctx0);
inp->self_k_rot = mctx_cur->get_base()->get_k_rot();
inp->self_v_rot = mctx_cur->get_base()->get_v_rot();
inp->self_k_rot_swa = mctx_cur->get_swa()->build_input_k_rot(ctx0);
inp->self_v_rot_swa = mctx_cur->get_swa()->build_input_v_rot(ctx0);
inp->self_k_rot_swa = mctx_cur->get_swa()->get_k_rot();
inp->self_v_rot_swa = mctx_cur->get_swa()->get_v_rot();
return (llm_graph_input_attn_kv_iswa *) res->add_input(std::move(inp));
}
@@ -3874,9 +3797,9 @@ llm_graph_input_attn_k_iswa * llm_graph_context::build_attn_inp_k_iswa() const {
inp->self_kq_mask_swa_cnv = inp->self_kq_mask_swa;
}
inp->self_k_rot = mctx_cur->get_base()->build_input_k_rot(ctx0);
inp->self_k_rot = mctx_cur->get_base()->get_k_rot();
inp->self_k_rot_swa = mctx_cur->get_swa()->build_input_k_rot(ctx0);
inp->self_k_rot_swa = mctx_cur->get_swa()->get_k_rot();
return (llm_graph_input_attn_k_iswa *) res->add_input(std::move(inp));
}
@@ -3895,15 +3818,15 @@ llm_graph_input_dsv4 * llm_graph_context::build_inp_dsv4() const {
inp_raw->self_kq_mask = dsv4_build_raw_kq_mask(ctx0, raw_ctx, ubatch, cparams, n_stream);
inp_raw->self_kq_mask_cnv = inp_raw->self_kq_mask;
inp_raw->self_k_rot = raw_ctx->build_input_k_rot(ctx0);
inp_raw->self_k_rot = raw_ctx->get_k_rot();
auto inp = std::make_unique<llm_graph_input_dsv4>(cparams, std::move(inp_raw), mctx_cur);
dsv4_build_comp_inputs(ctx0, inp->inp_csa, mctx_cur->get_csa_plan(ubatch), "csa", cparams, n_stream);
dsv4_build_comp_inputs(ctx0, inp->inp_hca, mctx_cur->get_hca_plan(ubatch), "hca", cparams, n_stream);
dsv4_build_comp_inputs(ctx0, inp->inp_lid, mctx_cur->get_lid_plan(ubatch), "lid", cparams, n_stream);
inp->inp_csa.k_rot = mctx_cur->get_csa()->build_input_k_rot(ctx0);
inp->inp_hca.k_rot = mctx_cur->get_hca()->build_input_k_rot(ctx0);
inp->inp_lid.k_rot = mctx_cur->get_lid()->build_input_k_rot(ctx0);
inp->inp_csa.k_rot = mctx_cur->get_csa()->get_k_rot();
inp->inp_hca.k_rot = mctx_cur->get_hca()->get_k_rot();
inp->inp_lid.k_rot = mctx_cur->get_lid()->get_k_rot();
return (llm_graph_input_dsv4 *) res->add_input(std::move(inp));
}
@@ -4088,6 +4011,12 @@ llm_graph_input_mem_hybrid_iswa * llm_graph_context::build_inp_mem_hybrid_iswa()
inp_attn->self_kq_mask_swa_cnv = inp_attn->self_kq_mask_swa;
}
inp_attn->self_k_rot = attn_ctx->get_base()->get_k_rot();
inp_attn->self_v_rot = attn_ctx->get_base()->get_v_rot();
inp_attn->self_k_rot_swa = attn_ctx->get_swa()->get_k_rot();
inp_attn->self_v_rot_swa = attn_ctx->get_swa()->get_v_rot();
auto inp = std::make_unique<llm_graph_input_mem_hybrid_iswa>(cparams, std::move(inp_attn), std::move(inp_rs), mctx_cur);
return (llm_graph_input_mem_hybrid_iswa *) res->add_input(std::move(inp));