llama: fix kv shifting, add tests, add per-model build_rope (#31683)
* llama: fix kv shifting and add tests * llama: add graph build_rope (#31693) * llama: add graph build_rope * improve * migrate new models * update review skill * fix ci * add --kv-shift to CI
This commit is contained in:
1 parent
b1c902f026
commit
6023692b1e
144 files changed
+1154
-1847
No files matched your search
+12
-11
@@ -145,6 +145,14 @@ void llama_model_dots3note::load_arch_tensors(llama_model_loader & ml) {
|
||||
}
|
||||
}
|
||||
|
||||
llama_rope_params llama_model_dots3note::get_rope_params(const llama_cparams & cparams, int il) const {
|
||||
auto res = llama_model::get_rope_params(cparams, il);
|
||||
|
||||
res.freq_scale = cparams.rope_freq_scale;
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
std::unique_ptr<llm_graph_context> llama_model_dots3note::build_arch_graph(const llm_graph_params & params) const {
|
||||
return std::make_unique<graph>(params);
|
||||
}
|
||||
@@ -186,7 +194,6 @@ llama_model_dots3note::graph::graph(const llm_graph_params & params) :
|
||||
const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope;
|
||||
|
||||
const float kq_scale = 1.0f/sqrtf(float(n_embd_head_k_mla));
|
||||
const float freq_base_l = model.get_rope_freq_base(cparams, il);
|
||||
|
||||
// norm
|
||||
cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
|
||||
@@ -211,9 +218,7 @@ llama_model_dots3note::graph::graph(const llm_graph_params & params) :
|
||||
|
||||
// {n_embd_indexer_head, n_indexer_head, n_tokens}
|
||||
indexer_q = ggml_reshape_3d(ctx0, indexer_q, n_embd_indexer_head, n_indexer_head, n_tokens);
|
||||
indexer_q = ggml_rope_ext(ctx0, indexer_q, inp_pos, nullptr, n_rot,
|
||||
LLAMA_ROPE_TYPE_NEOX, n_ctx_orig, freq_base, freq_scale,
|
||||
ext_factor, attn_factor, beta_fast, beta_slow);
|
||||
indexer_q = build_rope_indexer(indexer_q, inp_pos, il);
|
||||
cb(indexer_q, "indexer_q", il);
|
||||
|
||||
ggml_tensor * indexer_k = ggml_mul_mat(ctx0, model.layers[il].indexer_attn_k, cur);
|
||||
@@ -224,9 +229,7 @@ llama_model_dots3note::graph::graph(const llm_graph_params & params) :
|
||||
|
||||
// {n_embd_indexer_head, 1, n_tokens}
|
||||
indexer_k = ggml_reshape_3d(ctx0, indexer_k, n_embd_indexer_head, 1, n_tokens);
|
||||
indexer_k = ggml_rope_ext(ctx0, indexer_k, inp_pos, nullptr, n_rot,
|
||||
LLAMA_ROPE_TYPE_NEOX, n_ctx_orig, freq_base, freq_scale,
|
||||
ext_factor, attn_factor, beta_fast, beta_slow);
|
||||
indexer_k = build_rope_indexer(indexer_k, inp_pos, il);
|
||||
cb(indexer_k, "indexer_k", il);
|
||||
|
||||
// perform Hadamard transform on indexer q and k
|
||||
@@ -332,12 +335,10 @@ llama_model_dots3note::graph::graph(const llm_graph_params & params) :
|
||||
k_pe = build_norm(k_pe, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il);
|
||||
cb(k_pe, "k_pe", il);
|
||||
|
||||
q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale,
|
||||
ext_factor, attn_factor, beta_fast, beta_slow);
|
||||
q_pe = build_rope(q_pe, inp_pos, il);
|
||||
cb(q_pe, "q_pe", il);
|
||||
|
||||
k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale,
|
||||
ext_factor, attn_factor, beta_fast, beta_slow);
|
||||
k_pe = build_rope(k_pe, inp_pos, il);
|
||||
cb(k_pe, "k_pe", il);
|
||||
|
||||
kv_cmpr = build_norm(kv_cmpr, model.layers[il].attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);
|
||||
|
||||
Reference in new issue
Block a user