llama: fix kv shifting, add tests, add per-model build_rope (#31683)
* llama: fix kv shifting and add tests * llama: add graph build_rope (#31693) * llama: add graph build_rope * improve * migrate new models * update review skill * fix ci * add --kv-shift to CI
This commit is contained in:
1 parent
b1c902f026
commit
6023692b1e
144 files changed
+1154
-1847
No files matched your search
+10
-11
@@ -40,6 +40,14 @@ void llama_model_maincoder::load_arch_tensors(llama_model_loader &) {
|
||||
}
|
||||
}
|
||||
|
||||
llama_rope_params llama_model_maincoder::get_rope_params(const llama_cparams & cparams, int il) const {
|
||||
auto res = llama_model::get_rope_params(cparams, il);
|
||||
|
||||
res.k_norm = layers[il].attn_k_norm;
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
std::unique_ptr<llm_graph_context> llama_model_maincoder::build_arch_graph(const llm_graph_params & params) const {
|
||||
return std::make_unique<graph>(params);
|
||||
}
|
||||
@@ -77,17 +85,8 @@ llama_model_maincoder::graph::graph(const llm_graph_params & params) : llm_graph
|
||||
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
|
||||
n_embd_head, n_head, n_head_kv, il);
|
||||
|
||||
Qcur = ggml_rope_ext(
|
||||
ctx0, Qcur, inp_pos, nullptr,
|
||||
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
||||
ext_factor, attn_factor, beta_fast, beta_slow
|
||||
);
|
||||
|
||||
Kcur = ggml_rope_ext(
|
||||
ctx0, Kcur, inp_pos, nullptr,
|
||||
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
||||
ext_factor, attn_factor, beta_fast, beta_slow
|
||||
);
|
||||
Qcur = build_rope(Qcur, inp_pos, il);
|
||||
Kcur = build_rope(Kcur, inp_pos, il);
|
||||
|
||||
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
|
||||
cb(Qcur, "Qcur_normed", il);
|
||||
|
||||
Reference in new issue
Block a user