model: apply .scale tensors automatically + fix correctness (#32669)
* model: apply .scale tensors automatically * rm build_moe_ffn "_s" overload * use llama_prec_policy * fix ci * build_lora_mm_id: rm w_s
This commit is contained in:
1 parent
e50c254a22
commit
19bc41dbb4
39 files changed
+164
-290
No files matched your search
+22
-112
@@ -1810,16 +1810,21 @@ ggml_tensor * llm_graph_context::build_lora_mm(
|
||||
ggml_tensor * w,
|
||||
ggml_tensor * cur,
|
||||
ggml_tensor * w_s) const {
|
||||
if (!w_s) {
|
||||
w_s = model.get_w_scale(w);
|
||||
}
|
||||
|
||||
// a Hadamard-folded weight reads the transformed input; a LoRA delta lives in the primal basis and reads `cur`
|
||||
ggml_tensor * cur_mm = build_hadamard_input(w, cur);
|
||||
ggml_tensor * res = ggml_mul_mat(ctx0, w, cur_mm);
|
||||
|
||||
prec_policy.apply(res);
|
||||
|
||||
if (w->type == GGML_TYPE_NVFP4) {
|
||||
ggml_prec_set_acc(res, GGML_PREC_BF16);
|
||||
}
|
||||
|
||||
|
||||
prec_policy.apply(res);
|
||||
|
||||
if (w_s) {
|
||||
res = ggml_mul(ctx0, res, w_s);
|
||||
}
|
||||
@@ -1886,9 +1891,10 @@ ggml_tensor * llm_graph_context::build_lora_mm_id(
|
||||
ggml_tensor * w, // ggml_tensor * as
|
||||
ggml_tensor * cur, // ggml_tensor * b
|
||||
ggml_tensor * ids,
|
||||
ggml_tensor * w_s,
|
||||
ggml_tensor * slots,
|
||||
bool cached) const {
|
||||
ggml_tensor * w_s = model.get_w_scale(w);
|
||||
|
||||
// as in build_lora_mm: the folded weight reads the transformed input, the LoRA delta reads `cur`
|
||||
ggml_tensor * cur_mm = build_hadamard_input(w, cur);
|
||||
// the experts in the MoE cache are selected by their slots
|
||||
@@ -1896,12 +1902,13 @@ ggml_tensor * llm_graph_context::build_lora_mm_id(
|
||||
ggml_mul_mat_id(ctx0, w, cur_mm, ids) :
|
||||
ggml_mul_mat_id(ctx0, cached ? moe_cache->get_experts(w) : w, cur_mm, slots);
|
||||
|
||||
prec_policy.apply(res);
|
||||
|
||||
if (w->type == GGML_TYPE_NVFP4) {
|
||||
ggml_prec_set_acc(res, GGML_PREC_BF16);
|
||||
}
|
||||
|
||||
// after the NVFP4 default, so that the policy can request a higher precision
|
||||
prec_policy.apply(res);
|
||||
|
||||
if (w_s) {
|
||||
const int64_t n_expert = w_s->ne[0];
|
||||
const int64_t n_tokens = cur->ne[2];
|
||||
@@ -2115,34 +2122,7 @@ ggml_tensor * llm_graph_context::build_ffn(
|
||||
int il) const {
|
||||
const llm_arch arch = model.arch;
|
||||
|
||||
// NVFP4 support is currently restricted to
|
||||
// 1) LORA absence (*_s would be applied after LORA residual, which is incorrect)
|
||||
// 2) bias absense (*_s would be applied after bias addition, which is incorrect)
|
||||
// TODO: disambiguate LLM-architectural scales (which use *_s) from NVFP4 scale_2 (which also uses *_s currently)
|
||||
auto has_lora = [this](ggml_tensor * w) {
|
||||
if (!w) {
|
||||
return false;
|
||||
}
|
||||
for (const auto & lora : *loras) {
|
||||
if (lora.first->get_weight(w) != nullptr) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
auto is_scaled_low_precision = [](ggml_tensor * w) {
|
||||
return w && (w->type == GGML_TYPE_NVFP4 || w->type == GGML_TYPE_F8_E4M3);
|
||||
};
|
||||
|
||||
GGML_ASSERT(!up_s || !up_b || !is_scaled_low_precision(up));
|
||||
GGML_ASSERT(!gate_s || !gate_b || !is_scaled_low_precision(gate));
|
||||
GGML_ASSERT(!down_s || !down_b || !is_scaled_low_precision(down));
|
||||
GGML_ASSERT(!up_s || !is_scaled_low_precision(up) || !has_lora(up));
|
||||
GGML_ASSERT(!gate_s || !is_scaled_low_precision(gate) || !has_lora(gate));
|
||||
GGML_ASSERT(!down_s || !is_scaled_low_precision(down) || !has_lora(down));
|
||||
|
||||
ggml_tensor * tmp = up ? build_lora_mm(up, cur) : cur;
|
||||
ggml_tensor * tmp = up ? build_lora_mm(up, cur, up_s) : cur;
|
||||
cb(tmp, "ffn_up", il);
|
||||
|
||||
if (up_b) {
|
||||
@@ -2150,21 +2130,16 @@ ggml_tensor * llm_graph_context::build_ffn(
|
||||
cb(tmp, "ffn_up_b", il);
|
||||
}
|
||||
|
||||
if (up_s) {
|
||||
tmp = ggml_mul(ctx0, tmp, up_s);
|
||||
cb(tmp, "ffn_up_s", il);
|
||||
}
|
||||
|
||||
if (gate) {
|
||||
switch (type_gate) {
|
||||
case LLM_FFN_SEQ:
|
||||
{
|
||||
cur = build_lora_mm(gate, tmp);
|
||||
cur = build_lora_mm(gate, tmp, gate_s);
|
||||
cb(cur, "ffn_gate", il);
|
||||
} break;
|
||||
case LLM_FFN_PAR:
|
||||
{
|
||||
cur = build_lora_mm(gate, cur);
|
||||
cur = build_lora_mm(gate, cur, gate_s);
|
||||
cb(cur, "ffn_gate", il);
|
||||
} break;
|
||||
}
|
||||
@@ -2174,11 +2149,6 @@ ggml_tensor * llm_graph_context::build_ffn(
|
||||
cb(cur, "ffn_gate_b", il);
|
||||
}
|
||||
|
||||
if (gate_s) {
|
||||
cur = ggml_mul(ctx0, cur, gate_s);
|
||||
cb(cur, "ffn_gate_s", il);
|
||||
}
|
||||
|
||||
} else {
|
||||
cur = tmp;
|
||||
}
|
||||
@@ -2300,11 +2270,7 @@ ggml_tensor * llm_graph_context::build_ffn(
|
||||
}
|
||||
|
||||
if (down) {
|
||||
cur = build_lora_mm(down, cur);
|
||||
if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) {
|
||||
// GLM4, GLM4_MOE, and JAIS2 have numerical issues with F16 accumulators
|
||||
ggml_prec_set_acc(cur, GGML_PREC_BF16);
|
||||
}
|
||||
cur = build_lora_mm(down, cur, down_s);
|
||||
}
|
||||
|
||||
if (down_b) {
|
||||
@@ -2315,11 +2281,6 @@ ggml_tensor * llm_graph_context::build_ffn(
|
||||
cur = ggml_add(ctx0, cur, down_b);
|
||||
}
|
||||
|
||||
if (down_s) {
|
||||
cur = ggml_mul(ctx0, cur, down_s);
|
||||
cb(cur, "ffn_down_s", il);
|
||||
}
|
||||
|
||||
return cur;
|
||||
}
|
||||
|
||||
@@ -2339,9 +2300,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
|
||||
int il,
|
||||
ggml_tensor * probs_in,
|
||||
ggml_tensor * gate_up_exps,
|
||||
ggml_tensor * up_exps_s,
|
||||
ggml_tensor * gate_exps_s,
|
||||
ggml_tensor * down_exps_s,
|
||||
ggml_tensor * selected_experts_in,
|
||||
ggml_tensor * post_exp_norm,
|
||||
float post_exp_norm_eps) const {
|
||||
@@ -2362,9 +2320,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
|
||||
probs_in,
|
||||
gate_up_exps,
|
||||
/* gate_up_exps_b */ nullptr,
|
||||
up_exps_s,
|
||||
gate_exps_s,
|
||||
down_exps_s,
|
||||
selected_experts_in,
|
||||
post_exp_norm,
|
||||
post_exp_norm_eps
|
||||
@@ -2392,9 +2347,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
|
||||
ggml_tensor * probs_in,
|
||||
ggml_tensor * gate_up_exps,
|
||||
ggml_tensor * gate_up_exps_b,
|
||||
ggml_tensor * up_exps_s,
|
||||
ggml_tensor * gate_exps_s,
|
||||
ggml_tensor * down_exps_s,
|
||||
ggml_tensor * selected_experts_in,
|
||||
ggml_tensor * post_exp_norm,
|
||||
float post_exp_norm_eps) const {
|
||||
@@ -2407,10 +2359,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
|
||||
|
||||
if (probs_in == nullptr) {
|
||||
logits = build_lora_mm(gate_inp, cur); // [n_expert, n_tokens]
|
||||
if (gating_op == LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS || hparams.expert_router_f32) {
|
||||
// the reference computes the router in F32: https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/blob/60d8d70770c6776ff598c94bb586a859a38244f1/inference/model.py#L565
|
||||
ggml_prec_set_acc(logits, GGML_PREC_F32);
|
||||
}
|
||||
cb(logits, "ffn_moe_logits", il);
|
||||
} else {
|
||||
logits = probs_in;
|
||||
@@ -2583,13 +2531,9 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
|
||||
|
||||
if (gate_up_exps) {
|
||||
// merged gate_up path: one mul_mat_id, then split into gate and up views
|
||||
ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, cur, ids, up_exps_s, mm_ids, cached); // [n_ff*2, n_expert_used, n_tokens]
|
||||
ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, cur, ids, mm_ids, cached); // [n_ff*2, n_expert_used, n_tokens]
|
||||
cb(gate_up, "ffn_moe_gate_up", il);
|
||||
|
||||
if (up_exps_s) {
|
||||
cb(gate_up, "ffn_moe_gate_up_scaled", il);
|
||||
}
|
||||
|
||||
if (gate_up_exps_b) {
|
||||
gate_up = ggml_add_id(ctx0, gate_up, gate_up_exps_b, ids);
|
||||
cb(gate_up, "ffn_moe_gate_up_biased", il);
|
||||
@@ -2602,29 +2546,21 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
|
||||
cb(up, "ffn_moe_up", il);
|
||||
} else {
|
||||
// separate gate and up path
|
||||
up = build_lora_mm_id(up_exps, cur, ids, up_exps_s, mm_ids, cached); // [n_ff, n_expert_used, n_tokens]
|
||||
up = build_lora_mm_id(up_exps, cur, ids, mm_ids, cached); // [n_ff, n_expert_used, n_tokens]
|
||||
cb(up, "ffn_moe_up", il);
|
||||
|
||||
if (up_exps_s) {
|
||||
cb(up, "ffn_moe_up_scaled", il);
|
||||
}
|
||||
|
||||
if (up_exps_b) {
|
||||
up = ggml_add_id(ctx0, up, up_exps_b, ids);
|
||||
cb(up, "ffn_moe_up_biased", il);
|
||||
}
|
||||
|
||||
if (gate_exps) {
|
||||
cur = build_lora_mm_id(gate_exps, cur, ids, gate_exps_s, mm_ids, cached); // [n_ff, n_expert_used, n_tokens]
|
||||
cur = build_lora_mm_id(gate_exps, cur, ids, mm_ids, cached); // [n_ff, n_expert_used, n_tokens]
|
||||
cb(cur, "ffn_moe_gate", il);
|
||||
} else {
|
||||
cur = up;
|
||||
}
|
||||
|
||||
if (gate_exps_s) {
|
||||
cb(cur, "ffn_moe_gate_scaled", il);
|
||||
}
|
||||
|
||||
if (gate_exps_b) {
|
||||
cur = ggml_add_id(ctx0, cur, gate_exps_b, ids);
|
||||
cb(cur, "ffn_moe_gate_biased", il);
|
||||
@@ -2727,7 +2663,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
|
||||
cur = amax_src1.res;
|
||||
}
|
||||
|
||||
ggml_tensor * experts = build_lora_mm_id(down_exps, cur, ids, down_exps_s, mm_ids, cached); // [n_embd, n_expert_used, n_tokens]
|
||||
ggml_tensor * experts = build_lora_mm_id(down_exps, cur, ids, mm_ids, cached); // [n_embd, n_expert_used, n_tokens]
|
||||
//ggml_prec_set_src(experts, GGML_PREC_BF16, 1); // (see TODO above)
|
||||
cb(experts, "ffn_moe_down", il);
|
||||
|
||||
@@ -2745,10 +2681,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
|
||||
return (*static_cast<decltype(build_experts) *>(user_data))(cur, ids, mm_ids, cached);
|
||||
}, &build_experts);
|
||||
|
||||
if (down_exps_s) {
|
||||
cb(experts, "ffn_moe_down_scaled", il);
|
||||
}
|
||||
|
||||
if (down_exps_b) {
|
||||
experts = ggml_add_id(ctx0, experts, down_exps_b, selected_experts);
|
||||
cb(experts, "ffn_moe_down_biased", il);
|
||||
@@ -3485,8 +3417,6 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
ggml_tensor * v_mla, // TODO: remove
|
||||
float kq_scale,
|
||||
int il) const {
|
||||
const llm_arch arch = model.arch;
|
||||
|
||||
GGML_ASSERT(v_mla == nullptr);
|
||||
|
||||
if (inp->self_k_rot[il]) {
|
||||
@@ -3543,16 +3473,7 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
}
|
||||
|
||||
if (wo) {
|
||||
if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) {
|
||||
// GLM4, GLM4_MOE, and JAIS2 have numerical issues with F16 accumulators
|
||||
cur = build_lora_mm(wo, cur);
|
||||
ggml_prec_set_acc(cur, GGML_PREC_BF16);
|
||||
if (wo_s) {
|
||||
cur = ggml_mul(ctx0, cur, wo_s);
|
||||
}
|
||||
} else {
|
||||
cur = build_lora_mm(wo, cur, wo_s);
|
||||
}
|
||||
cur = build_lora_mm(wo, cur, wo_s);
|
||||
}
|
||||
|
||||
if (wo_b) {
|
||||
@@ -3603,8 +3524,6 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
ggml_tensor * v_mla,
|
||||
float kq_scale,
|
||||
int il) const {
|
||||
const llm_arch arch = model.arch;
|
||||
|
||||
// these nodes are added to the graph together so that they are not reordered
|
||||
// by doing so, the number of splits in the graph is reduced
|
||||
// expand k later to enable rope fusion which directly writes into k-v cache
|
||||
@@ -3632,16 +3551,7 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
cb(cur, "kqv_out", il);
|
||||
|
||||
if (wo) {
|
||||
if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE) {
|
||||
// GLM4 and GLM4_MOE have numerical issues with F16 accumulators
|
||||
cur = build_lora_mm(wo, cur);
|
||||
ggml_prec_set_acc(cur, GGML_PREC_BF16);
|
||||
if (wo_s) {
|
||||
cur = ggml_mul(ctx0, cur, wo_s);
|
||||
}
|
||||
} else {
|
||||
cur = build_lora_mm(wo, cur, wo_s);
|
||||
}
|
||||
cur = build_lora_mm(wo, cur, wo_s);
|
||||
}
|
||||
|
||||
if (wo_b) {
|
||||
|
||||
Reference in new issue
Block a user