Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

convert : add --fuse-qkv flag to fuse Q/K/V into QKV during HF-to-GGUF conversion (#22780)

Browse Source
This commit is contained in:
PikaPikachu
2026-09-07 00:47:05 +08:00
committed by GitHub
parent 5fdfa62829
commit 465e49b9ce
20 changed files with 396 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-graph.cpp
+81 -29
View File
@@ -1623,8 +1623,26 @@ llm_graph_qkv llm_graph_context::build_qkv(
int64_t n_head,
int64_t n_head_kv,
int il) const {
const int64_t n_embd_q = n_embd_head * n_head;
const int64_t n_embd_kv = n_embd_head * n_head_kv;
return build_qkv(layer, cur,
n_embd_head, n_head,
n_embd_head, n_head_kv,
n_embd_head, n_head_kv,
il);
}
llm_graph_qkv llm_graph_context::build_qkv(
const llama_layer & layer,
ggml_tensor * cur,
int64_t n_embd_head_q,
int64_t n_head_q,
int64_t n_embd_head_k,
int64_t n_head_k,
int64_t n_embd_head_v,
int64_t n_head_v,
int il,
bool reshape) const {
const int64_t n_embd_q = n_embd_head_q * n_head_q;
const int64_t n_embd_k = n_embd_head_k * n_head_k;
ggml_tensor * Qcur, * Kcur, * Vcur;
@@ -1635,59 +1653,93 @@ llm_graph_qkv llm_graph_context::build_qkv(
if (layer.wqkv_b) {
qkv = ggml_add(ctx0, qkv, layer.wqkv_b);
cb(qkv, "wqkv_b", il);
} else if (layer.wq_b && layer.wk_b && layer.wv_b) {
// Fused weights may coexist with separate Q/K/V biases in legacy or custom GGUFs.
ggml_tensor * qkv_b = ggml_concat(ctx0, ggml_concat(ctx0, layer.wq_b, layer.wk_b, 0), layer.wv_b, 0);
qkv = ggml_add(ctx0, qkv, qkv_b);
cb(qkv, "wqkv_b", il);
}
if (hparams.f_clamp_kqv > 0.0f) {
if (reshape && hparams.f_clamp_kqv > 0.0f) {
qkv = ggml_clamp(ctx0, qkv, -hparams.f_clamp_kqv, hparams.f_clamp_kqv);
cb(qkv, "wqkv_clamped", il);
}
Qcur = ggml_view_3d(ctx0, qkv, n_embd_head, n_head, n_tokens,
ggml_row_size(qkv->type, n_embd_head), qkv->nb[1], 0);
Kcur = ggml_view_3d(ctx0, qkv, n_embd_head, n_head_kv, n_tokens,
ggml_row_size(qkv->type, n_embd_head), qkv->nb[1],
ggml_row_size(qkv->type, n_embd_q));
Vcur = ggml_view_3d(ctx0, qkv, n_embd_head, n_head_kv, n_tokens,
ggml_row_size(qkv->type, n_embd_head), qkv->nb[1],
ggml_row_size(qkv->type, n_embd_q + n_embd_kv));
if (reshape) {
Qcur = ggml_view_3d(ctx0, qkv, n_embd_head_q, n_head_q, n_tokens,
ggml_row_size(qkv->type, n_embd_head_q), qkv->nb[1], 0);
Kcur = ggml_view_3d(ctx0, qkv, n_embd_head_k, n_head_k, n_tokens,
ggml_row_size(qkv->type, n_embd_head_k), qkv->nb[1],
ggml_row_size(qkv->type, n_embd_q));
Vcur = ggml_view_3d(ctx0, qkv, n_embd_head_v, n_head_v, n_tokens,
ggml_row_size(qkv->type, n_embd_head_v), qkv->nb[1],
ggml_row_size(qkv->type, n_embd_q + n_embd_k));
} else {
Qcur = ggml_view_2d(ctx0, qkv, n_embd_q, n_tokens, qkv->nb[1], 0);
Kcur = ggml_view_2d(ctx0, qkv, n_embd_k, n_tokens, qkv->nb[1],
ggml_row_size(qkv->type, n_embd_q));
Vcur = ggml_view_2d(ctx0, qkv, n_embd_head_v * n_head_v, n_tokens, qkv->nb[1],
ggml_row_size(qkv->type, n_embd_q + n_embd_k));
}
if (!reshape) {
Qcur = ggml_cont(ctx0, Qcur);
Kcur = ggml_cont(ctx0, Kcur);
Vcur = ggml_cont(ctx0, Vcur);
}
} else {
// separate Q/K/V path
Qcur = build_lora_mm(layer.wq, cur, layer.wq_s);
cb(Qcur, "Qcur", il);
if (layer.wq_b) {
Qcur = ggml_add(ctx0, Qcur, layer.wq_b);
if (reshape) {
cb(Qcur, "Qcur", il);
}
if (hparams.f_clamp_kqv > 0.0f) {
if (layer.wq_b) {
Qcur = ggml_add(ctx0, Qcur, layer.wq_b);
if (reshape) {
cb(Qcur, "Qcur", il);
}
}
if (reshape && hparams.f_clamp_kqv > 0.0f) {
Qcur = ggml_clamp(ctx0, Qcur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv);
cb(Qcur, "Qcur_clamped", il);
}
Kcur = build_lora_mm(layer.wk, cur, layer.wk_s);
cb(Kcur, "Kcur", il);
if (layer.wk_b) {
Kcur = ggml_add(ctx0, Kcur, layer.wk_b);
if (reshape) {
cb(Kcur, "Kcur", il);
}
if (hparams.f_clamp_kqv > 0.0f) {
if (layer.wk_b) {
Kcur = ggml_add(ctx0, Kcur, layer.wk_b);
if (reshape) {
cb(Kcur, "Kcur", il);
}
}
if (reshape && hparams.f_clamp_kqv > 0.0f) {
Kcur = ggml_clamp(ctx0, Kcur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv);
cb(Kcur, "Kcur_clamped", il);
}
Vcur = build_lora_mm(layer.wv, cur, layer.wv_s);
cb(Vcur, "Vcur", il);
if (layer.wv_b) {
Vcur = ggml_add(ctx0, Vcur, layer.wv_b);
if (reshape) {
cb(Vcur, "Vcur", il);
}
if (hparams.f_clamp_kqv > 0.0f) {
if (layer.wv_b) {
Vcur = ggml_add(ctx0, Vcur, layer.wv_b);
if (reshape) {
cb(Vcur, "Vcur", il);
}
}
if (reshape && hparams.f_clamp_kqv > 0.0f) {
Vcur = ggml_clamp(ctx0, Vcur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv);
cb(Vcur, "Vcur_clamped", il);
}
Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
if (reshape) {
Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head_q, n_head_q, n_tokens);
Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_k, n_tokens);
Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head_v, n_head_v, n_tokens);
}
}
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
cb(Vcur, "Vcur", il);
if (reshape) {
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
cb(Vcur, "Vcur", il);
}
return { Qcur, Kcur, Vcur };
}