llama: fix kv shifting, add tests, add per-model build_rope (#31683)

* llama: fix kv shifting and add tests

* llama: add graph build_rope (#31693)

* llama: add graph build_rope

* improve

* migrate new models

* update review skill

* fix ci

* add --kv-shift to CI
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-11-05 21:10:36 +01:00
1 parent b1c902f026
commit 6023692b1e
144 files changed
+1154 -1847

No files matched your search

+12 -11
View File
@@ -145,6 +145,14 @@ void llama_model_dots3note::load_arch_tensors(llama_model_loader & ml) {
}
}
llama_rope_params llama_model_dots3note::get_rope_params(const llama_cparams & cparams, int il) const {
auto res = llama_model::get_rope_params(cparams, il);
res.freq_scale = cparams.rope_freq_scale;
return res;
}
std::unique_ptr<llm_graph_context> llama_model_dots3note::build_arch_graph(const llm_graph_params & params) const {
return std::make_unique<graph>(params);
}
@@ -186,7 +194,6 @@ llama_model_dots3note::graph::graph(const llm_graph_params & params) :
const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope;
const float kq_scale = 1.0f/sqrtf(float(n_embd_head_k_mla));
const float freq_base_l = model.get_rope_freq_base(cparams, il);
// norm
cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
@@ -211,9 +218,7 @@ llama_model_dots3note::graph::graph(const llm_graph_params & params) :
// {n_embd_indexer_head, n_indexer_head, n_tokens}
indexer_q = ggml_reshape_3d(ctx0, indexer_q, n_embd_indexer_head, n_indexer_head, n_tokens);
indexer_q = ggml_rope_ext(ctx0, indexer_q, inp_pos, nullptr, n_rot,
LLAMA_ROPE_TYPE_NEOX, n_ctx_orig, freq_base, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow);
indexer_q = build_rope_indexer(indexer_q, inp_pos, il);
cb(indexer_q, "indexer_q", il);
ggml_tensor * indexer_k = ggml_mul_mat(ctx0, model.layers[il].indexer_attn_k, cur);
@@ -224,9 +229,7 @@ llama_model_dots3note::graph::graph(const llm_graph_params & params) :
// {n_embd_indexer_head, 1, n_tokens}
indexer_k = ggml_reshape_3d(ctx0, indexer_k, n_embd_indexer_head, 1, n_tokens);
indexer_k = ggml_rope_ext(ctx0, indexer_k, inp_pos, nullptr, n_rot,
LLAMA_ROPE_TYPE_NEOX, n_ctx_orig, freq_base, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow);
indexer_k = build_rope_indexer(indexer_k, inp_pos, il);
cb(indexer_k, "indexer_k", il);
// perform Hadamard transform on indexer q and k
@@ -332,12 +335,10 @@ llama_model_dots3note::graph::graph(const llm_graph_params & params) :
k_pe = build_norm(k_pe, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il);
cb(k_pe, "k_pe", il);
q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow);
q_pe = build_rope(q_pe, inp_pos, il);
cb(q_pe, "q_pe", il);
k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale,
ext_factor, attn_factor, beta_fast, beta_slow);
k_pe = build_rope(k_pe, inp_pos, il);
cb(k_pe, "k_pe", il);
kv_cmpr = build_norm(kv_cmpr, model.layers[il].attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);