model: apply .scale tensors automatically + fix correctness (#32669)
* model: apply .scale tensors automatically * rm build_moe_ffn "_s" overload * use llama_prec_policy * fix ci * build_lora_mm_id: rm w_s
This commit is contained in:
1 parent
e50c254a22
commit
19bc41dbb4
39 files changed
+164
-290
No files matched your search
@@ -2154,8 +2154,28 @@ static enum ggml_status ggml_backend_meta_graph_compute(ggml_backend_t backend,
|
||||
}
|
||||
};
|
||||
|
||||
skip_unrelated();
|
||||
if (id + 1 >= cgraph->n_nodes) {
|
||||
// Chain of MULs with MIRRORED src[1], returns false at the end of the graph
|
||||
auto absorb_muls = [&]() -> bool {
|
||||
while (true) {
|
||||
skip_unrelated();
|
||||
if (id + 1 >= cgraph->n_nodes) {
|
||||
return false;
|
||||
}
|
||||
ggml_tensor * next = cgraph->nodes[id+1];
|
||||
if (next->op == GGML_OP_MUL && next->src[0] == node &&
|
||||
ggml_backend_meta_get_split_state(next->src[1], false).axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED) {
|
||||
node = next;
|
||||
id++;
|
||||
idr = id;
|
||||
n_used = ggml_node_get_use_count(cgraph, id);
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// MULs before the PARTIAL bias (e.g. a per-expert weight scale): sum_j(s*p_j + b/n) == s*sum_j(p_j) + b
|
||||
if (!absorb_muls()) {
|
||||
return idr;
|
||||
}
|
||||
{
|
||||
@@ -2169,22 +2189,8 @@ static enum ggml_status ggml_backend_meta_graph_compute(ggml_backend_t backend,
|
||||
n_used = ggml_node_get_use_count(cgraph, id);
|
||||
}
|
||||
}
|
||||
// Chain of MULs with MIRRORED src[1]
|
||||
while (true) {
|
||||
skip_unrelated();
|
||||
if (id + 1 >= cgraph->n_nodes) {
|
||||
return idr;
|
||||
}
|
||||
ggml_tensor * next = cgraph->nodes[id+1];
|
||||
if (next->op == GGML_OP_MUL && next->src[0] == node &&
|
||||
ggml_backend_meta_get_split_state(next->src[1], false).axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED) {
|
||||
node = next;
|
||||
id++;
|
||||
idr = id;
|
||||
n_used = ggml_node_get_use_count(cgraph, id);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
if (!absorb_muls()) {
|
||||
return idr;
|
||||
}
|
||||
|
||||
if (n_used != node->ne[1] || id + 2*n_used-1 >= cgraph->n_nodes) {
|
||||
|
||||
+22
-112
@@ -1810,16 +1810,21 @@ ggml_tensor * llm_graph_context::build_lora_mm(
|
||||
ggml_tensor * w,
|
||||
ggml_tensor * cur,
|
||||
ggml_tensor * w_s) const {
|
||||
if (!w_s) {
|
||||
w_s = model.get_w_scale(w);
|
||||
}
|
||||
|
||||
// a Hadamard-folded weight reads the transformed input; a LoRA delta lives in the primal basis and reads `cur`
|
||||
ggml_tensor * cur_mm = build_hadamard_input(w, cur);
|
||||
ggml_tensor * res = ggml_mul_mat(ctx0, w, cur_mm);
|
||||
|
||||
prec_policy.apply(res);
|
||||
|
||||
if (w->type == GGML_TYPE_NVFP4) {
|
||||
ggml_prec_set_acc(res, GGML_PREC_BF16);
|
||||
}
|
||||
|
||||
|
||||
prec_policy.apply(res);
|
||||
|
||||
if (w_s) {
|
||||
res = ggml_mul(ctx0, res, w_s);
|
||||
}
|
||||
@@ -1886,9 +1891,10 @@ ggml_tensor * llm_graph_context::build_lora_mm_id(
|
||||
ggml_tensor * w, // ggml_tensor * as
|
||||
ggml_tensor * cur, // ggml_tensor * b
|
||||
ggml_tensor * ids,
|
||||
ggml_tensor * w_s,
|
||||
ggml_tensor * slots,
|
||||
bool cached) const {
|
||||
ggml_tensor * w_s = model.get_w_scale(w);
|
||||
|
||||
// as in build_lora_mm: the folded weight reads the transformed input, the LoRA delta reads `cur`
|
||||
ggml_tensor * cur_mm = build_hadamard_input(w, cur);
|
||||
// the experts in the MoE cache are selected by their slots
|
||||
@@ -1896,12 +1902,13 @@ ggml_tensor * llm_graph_context::build_lora_mm_id(
|
||||
ggml_mul_mat_id(ctx0, w, cur_mm, ids) :
|
||||
ggml_mul_mat_id(ctx0, cached ? moe_cache->get_experts(w) : w, cur_mm, slots);
|
||||
|
||||
prec_policy.apply(res);
|
||||
|
||||
if (w->type == GGML_TYPE_NVFP4) {
|
||||
ggml_prec_set_acc(res, GGML_PREC_BF16);
|
||||
}
|
||||
|
||||
// after the NVFP4 default, so that the policy can request a higher precision
|
||||
prec_policy.apply(res);
|
||||
|
||||
if (w_s) {
|
||||
const int64_t n_expert = w_s->ne[0];
|
||||
const int64_t n_tokens = cur->ne[2];
|
||||
@@ -2115,34 +2122,7 @@ ggml_tensor * llm_graph_context::build_ffn(
|
||||
int il) const {
|
||||
const llm_arch arch = model.arch;
|
||||
|
||||
// NVFP4 support is currently restricted to
|
||||
// 1) LORA absence (*_s would be applied after LORA residual, which is incorrect)
|
||||
// 2) bias absense (*_s would be applied after bias addition, which is incorrect)
|
||||
// TODO: disambiguate LLM-architectural scales (which use *_s) from NVFP4 scale_2 (which also uses *_s currently)
|
||||
auto has_lora = [this](ggml_tensor * w) {
|
||||
if (!w) {
|
||||
return false;
|
||||
}
|
||||
for (const auto & lora : *loras) {
|
||||
if (lora.first->get_weight(w) != nullptr) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
auto is_scaled_low_precision = [](ggml_tensor * w) {
|
||||
return w && (w->type == GGML_TYPE_NVFP4 || w->type == GGML_TYPE_F8_E4M3);
|
||||
};
|
||||
|
||||
GGML_ASSERT(!up_s || !up_b || !is_scaled_low_precision(up));
|
||||
GGML_ASSERT(!gate_s || !gate_b || !is_scaled_low_precision(gate));
|
||||
GGML_ASSERT(!down_s || !down_b || !is_scaled_low_precision(down));
|
||||
GGML_ASSERT(!up_s || !is_scaled_low_precision(up) || !has_lora(up));
|
||||
GGML_ASSERT(!gate_s || !is_scaled_low_precision(gate) || !has_lora(gate));
|
||||
GGML_ASSERT(!down_s || !is_scaled_low_precision(down) || !has_lora(down));
|
||||
|
||||
ggml_tensor * tmp = up ? build_lora_mm(up, cur) : cur;
|
||||
ggml_tensor * tmp = up ? build_lora_mm(up, cur, up_s) : cur;
|
||||
cb(tmp, "ffn_up", il);
|
||||
|
||||
if (up_b) {
|
||||
@@ -2150,21 +2130,16 @@ ggml_tensor * llm_graph_context::build_ffn(
|
||||
cb(tmp, "ffn_up_b", il);
|
||||
}
|
||||
|
||||
if (up_s) {
|
||||
tmp = ggml_mul(ctx0, tmp, up_s);
|
||||
cb(tmp, "ffn_up_s", il);
|
||||
}
|
||||
|
||||
if (gate) {
|
||||
switch (type_gate) {
|
||||
case LLM_FFN_SEQ:
|
||||
{
|
||||
cur = build_lora_mm(gate, tmp);
|
||||
cur = build_lora_mm(gate, tmp, gate_s);
|
||||
cb(cur, "ffn_gate", il);
|
||||
} break;
|
||||
case LLM_FFN_PAR:
|
||||
{
|
||||
cur = build_lora_mm(gate, cur);
|
||||
cur = build_lora_mm(gate, cur, gate_s);
|
||||
cb(cur, "ffn_gate", il);
|
||||
} break;
|
||||
}
|
||||
@@ -2174,11 +2149,6 @@ ggml_tensor * llm_graph_context::build_ffn(
|
||||
cb(cur, "ffn_gate_b", il);
|
||||
}
|
||||
|
||||
if (gate_s) {
|
||||
cur = ggml_mul(ctx0, cur, gate_s);
|
||||
cb(cur, "ffn_gate_s", il);
|
||||
}
|
||||
|
||||
} else {
|
||||
cur = tmp;
|
||||
}
|
||||
@@ -2300,11 +2270,7 @@ ggml_tensor * llm_graph_context::build_ffn(
|
||||
}
|
||||
|
||||
if (down) {
|
||||
cur = build_lora_mm(down, cur);
|
||||
if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) {
|
||||
// GLM4, GLM4_MOE, and JAIS2 have numerical issues with F16 accumulators
|
||||
ggml_prec_set_acc(cur, GGML_PREC_BF16);
|
||||
}
|
||||
cur = build_lora_mm(down, cur, down_s);
|
||||
}
|
||||
|
||||
if (down_b) {
|
||||
@@ -2315,11 +2281,6 @@ ggml_tensor * llm_graph_context::build_ffn(
|
||||
cur = ggml_add(ctx0, cur, down_b);
|
||||
}
|
||||
|
||||
if (down_s) {
|
||||
cur = ggml_mul(ctx0, cur, down_s);
|
||||
cb(cur, "ffn_down_s", il);
|
||||
}
|
||||
|
||||
return cur;
|
||||
}
|
||||
|
||||
@@ -2339,9 +2300,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
|
||||
int il,
|
||||
ggml_tensor * probs_in,
|
||||
ggml_tensor * gate_up_exps,
|
||||
ggml_tensor * up_exps_s,
|
||||
ggml_tensor * gate_exps_s,
|
||||
ggml_tensor * down_exps_s,
|
||||
ggml_tensor * selected_experts_in,
|
||||
ggml_tensor * post_exp_norm,
|
||||
float post_exp_norm_eps) const {
|
||||
@@ -2362,9 +2320,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
|
||||
probs_in,
|
||||
gate_up_exps,
|
||||
/* gate_up_exps_b */ nullptr,
|
||||
up_exps_s,
|
||||
gate_exps_s,
|
||||
down_exps_s,
|
||||
selected_experts_in,
|
||||
post_exp_norm,
|
||||
post_exp_norm_eps
|
||||
@@ -2392,9 +2347,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
|
||||
ggml_tensor * probs_in,
|
||||
ggml_tensor * gate_up_exps,
|
||||
ggml_tensor * gate_up_exps_b,
|
||||
ggml_tensor * up_exps_s,
|
||||
ggml_tensor * gate_exps_s,
|
||||
ggml_tensor * down_exps_s,
|
||||
ggml_tensor * selected_experts_in,
|
||||
ggml_tensor * post_exp_norm,
|
||||
float post_exp_norm_eps) const {
|
||||
@@ -2407,10 +2359,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
|
||||
|
||||
if (probs_in == nullptr) {
|
||||
logits = build_lora_mm(gate_inp, cur); // [n_expert, n_tokens]
|
||||
if (gating_op == LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS || hparams.expert_router_f32) {
|
||||
// the reference computes the router in F32: https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/blob/60d8d70770c6776ff598c94bb586a859a38244f1/inference/model.py#L565
|
||||
ggml_prec_set_acc(logits, GGML_PREC_F32);
|
||||
}
|
||||
cb(logits, "ffn_moe_logits", il);
|
||||
} else {
|
||||
logits = probs_in;
|
||||
@@ -2583,13 +2531,9 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
|
||||
|
||||
if (gate_up_exps) {
|
||||
// merged gate_up path: one mul_mat_id, then split into gate and up views
|
||||
ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, cur, ids, up_exps_s, mm_ids, cached); // [n_ff*2, n_expert_used, n_tokens]
|
||||
ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, cur, ids, mm_ids, cached); // [n_ff*2, n_expert_used, n_tokens]
|
||||
cb(gate_up, "ffn_moe_gate_up", il);
|
||||
|
||||
if (up_exps_s) {
|
||||
cb(gate_up, "ffn_moe_gate_up_scaled", il);
|
||||
}
|
||||
|
||||
if (gate_up_exps_b) {
|
||||
gate_up = ggml_add_id(ctx0, gate_up, gate_up_exps_b, ids);
|
||||
cb(gate_up, "ffn_moe_gate_up_biased", il);
|
||||
@@ -2602,29 +2546,21 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
|
||||
cb(up, "ffn_moe_up", il);
|
||||
} else {
|
||||
// separate gate and up path
|
||||
up = build_lora_mm_id(up_exps, cur, ids, up_exps_s, mm_ids, cached); // [n_ff, n_expert_used, n_tokens]
|
||||
up = build_lora_mm_id(up_exps, cur, ids, mm_ids, cached); // [n_ff, n_expert_used, n_tokens]
|
||||
cb(up, "ffn_moe_up", il);
|
||||
|
||||
if (up_exps_s) {
|
||||
cb(up, "ffn_moe_up_scaled", il);
|
||||
}
|
||||
|
||||
if (up_exps_b) {
|
||||
up = ggml_add_id(ctx0, up, up_exps_b, ids);
|
||||
cb(up, "ffn_moe_up_biased", il);
|
||||
}
|
||||
|
||||
if (gate_exps) {
|
||||
cur = build_lora_mm_id(gate_exps, cur, ids, gate_exps_s, mm_ids, cached); // [n_ff, n_expert_used, n_tokens]
|
||||
cur = build_lora_mm_id(gate_exps, cur, ids, mm_ids, cached); // [n_ff, n_expert_used, n_tokens]
|
||||
cb(cur, "ffn_moe_gate", il);
|
||||
} else {
|
||||
cur = up;
|
||||
}
|
||||
|
||||
if (gate_exps_s) {
|
||||
cb(cur, "ffn_moe_gate_scaled", il);
|
||||
}
|
||||
|
||||
if (gate_exps_b) {
|
||||
cur = ggml_add_id(ctx0, cur, gate_exps_b, ids);
|
||||
cb(cur, "ffn_moe_gate_biased", il);
|
||||
@@ -2727,7 +2663,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
|
||||
cur = amax_src1.res;
|
||||
}
|
||||
|
||||
ggml_tensor * experts = build_lora_mm_id(down_exps, cur, ids, down_exps_s, mm_ids, cached); // [n_embd, n_expert_used, n_tokens]
|
||||
ggml_tensor * experts = build_lora_mm_id(down_exps, cur, ids, mm_ids, cached); // [n_embd, n_expert_used, n_tokens]
|
||||
//ggml_prec_set_src(experts, GGML_PREC_BF16, 1); // (see TODO above)
|
||||
cb(experts, "ffn_moe_down", il);
|
||||
|
||||
@@ -2745,10 +2681,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
|
||||
return (*static_cast<decltype(build_experts) *>(user_data))(cur, ids, mm_ids, cached);
|
||||
}, &build_experts);
|
||||
|
||||
if (down_exps_s) {
|
||||
cb(experts, "ffn_moe_down_scaled", il);
|
||||
}
|
||||
|
||||
if (down_exps_b) {
|
||||
experts = ggml_add_id(ctx0, experts, down_exps_b, selected_experts);
|
||||
cb(experts, "ffn_moe_down_biased", il);
|
||||
@@ -3485,8 +3417,6 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
ggml_tensor * v_mla, // TODO: remove
|
||||
float kq_scale,
|
||||
int il) const {
|
||||
const llm_arch arch = model.arch;
|
||||
|
||||
GGML_ASSERT(v_mla == nullptr);
|
||||
|
||||
if (inp->self_k_rot[il]) {
|
||||
@@ -3543,16 +3473,7 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
}
|
||||
|
||||
if (wo) {
|
||||
if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) {
|
||||
// GLM4, GLM4_MOE, and JAIS2 have numerical issues with F16 accumulators
|
||||
cur = build_lora_mm(wo, cur);
|
||||
ggml_prec_set_acc(cur, GGML_PREC_BF16);
|
||||
if (wo_s) {
|
||||
cur = ggml_mul(ctx0, cur, wo_s);
|
||||
}
|
||||
} else {
|
||||
cur = build_lora_mm(wo, cur, wo_s);
|
||||
}
|
||||
cur = build_lora_mm(wo, cur, wo_s);
|
||||
}
|
||||
|
||||
if (wo_b) {
|
||||
@@ -3603,8 +3524,6 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
ggml_tensor * v_mla,
|
||||
float kq_scale,
|
||||
int il) const {
|
||||
const llm_arch arch = model.arch;
|
||||
|
||||
// these nodes are added to the graph together so that they are not reordered
|
||||
// by doing so, the number of splits in the graph is reduced
|
||||
// expand k later to enable rope fusion which directly writes into k-v cache
|
||||
@@ -3632,16 +3551,7 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
cb(cur, "kqv_out", il);
|
||||
|
||||
if (wo) {
|
||||
if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE) {
|
||||
// GLM4 and GLM4_MOE have numerical issues with F16 accumulators
|
||||
cur = build_lora_mm(wo, cur);
|
||||
ggml_prec_set_acc(cur, GGML_PREC_BF16);
|
||||
if (wo_s) {
|
||||
cur = ggml_mul(ctx0, cur, wo_s);
|
||||
}
|
||||
} else {
|
||||
cur = build_lora_mm(wo, cur, wo_s);
|
||||
}
|
||||
cur = build_lora_mm(wo, cur, wo_s);
|
||||
}
|
||||
|
||||
if (wo_b) {
|
||||
|
||||
+1
-7
@@ -1179,6 +1179,7 @@ struct llm_graph_context {
|
||||
ggml_tensor * cur) const;
|
||||
|
||||
// do mat_mul, while optionally apply lora and per-tensor scale
|
||||
// w_s defaults to the weight scale of w
|
||||
ggml_tensor * build_lora_mm(
|
||||
ggml_tensor * w,
|
||||
ggml_tensor * cur,
|
||||
@@ -1190,7 +1191,6 @@ struct llm_graph_context {
|
||||
ggml_tensor * w, // ggml_tensor * as
|
||||
ggml_tensor * cur, // ggml_tensor * b
|
||||
ggml_tensor * ids,
|
||||
ggml_tensor * w_s = nullptr,
|
||||
ggml_tensor * slots = nullptr,
|
||||
bool cached = true) const;
|
||||
|
||||
@@ -1263,9 +1263,6 @@ struct llm_graph_context {
|
||||
int il,
|
||||
ggml_tensor * probs_in = nullptr,
|
||||
ggml_tensor * gate_up_exps = nullptr,
|
||||
ggml_tensor * up_exps_s = nullptr,
|
||||
ggml_tensor * gate_exps_s = nullptr,
|
||||
ggml_tensor * down_exps_s = nullptr,
|
||||
ggml_tensor * selected_experts_in = nullptr,
|
||||
ggml_tensor * post_exp_norm = nullptr,
|
||||
float post_exp_norm_eps = 1e-5f) const;
|
||||
@@ -1291,9 +1288,6 @@ struct llm_graph_context {
|
||||
ggml_tensor * probs_in = nullptr,
|
||||
ggml_tensor * gate_up_exps = nullptr,
|
||||
ggml_tensor * gate_up_exps_b = nullptr,
|
||||
ggml_tensor * up_exps_s = nullptr,
|
||||
ggml_tensor * gate_exps_s = nullptr,
|
||||
ggml_tensor * down_exps_s = nullptr,
|
||||
ggml_tensor * selected_experts_in = nullptr,
|
||||
ggml_tensor * post_exp_norm = nullptr,
|
||||
float post_exp_norm_eps = 1e-5f) const;
|
||||
|
||||
+52
-4
@@ -1410,12 +1410,19 @@ bool llama_prec_policy::apply(ggml_tensor * res) const {
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto it = prec_src1.find(res->src[0]);
|
||||
if (it == prec_src1.end()) {
|
||||
return false;
|
||||
bool applied = false;
|
||||
|
||||
const auto it_acc = prec_acc.find(res->src[0]);
|
||||
if (it_acc != prec_acc.end()) {
|
||||
applied |= ggml_prec_set_acc(res, it_acc->second);
|
||||
}
|
||||
|
||||
return ggml_prec_set_src(res, it->second, 1);
|
||||
const auto it_src1 = prec_src1.find(res->src[0]);
|
||||
if (it_src1 != prec_src1.end()) {
|
||||
applied |= ggml_prec_set_src(res, it_src1->second, 1);
|
||||
}
|
||||
|
||||
return applied;
|
||||
}
|
||||
|
||||
void llama_prec_policy::load(llama_model_loader & ml, const llama_model & model) {
|
||||
@@ -2044,6 +2051,42 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
|
||||
}
|
||||
}
|
||||
|
||||
// map each weight to its "<name>.scale" tensor: shape {1} for a 2D weight, {n_expert} for a 3D weight
|
||||
{
|
||||
auto ends_with = [](const std::string & str, const std::string & suffix) {
|
||||
return str.size() >= suffix.size() && str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0;
|
||||
};
|
||||
std::unordered_map<std::string, ggml_tensor *> weights;
|
||||
for (const auto & [name, cur] : tensors_by_name) {
|
||||
if (ends_with(name, ".weight")) {
|
||||
weights.emplace(name.substr(0, name.size() - 7), cur);
|
||||
}
|
||||
}
|
||||
for (const auto & [name, s] : tensors_by_name) {
|
||||
if (!ends_with(name, ".scale")) {
|
||||
continue;
|
||||
}
|
||||
const auto it = weights.find(name.substr(0, name.size() - 6));
|
||||
if (it == weights.end()) {
|
||||
continue;
|
||||
}
|
||||
ggml_tensor * w = it->second;
|
||||
const bool ok = w->ne[3] == 1 && ggml_nelements(s) == w->ne[2] && s->ne[0] == w->ne[2];
|
||||
if (ok) {
|
||||
w_scales.emplace(w, s);
|
||||
} else {
|
||||
LLAMA_LOG_DEBUG("%s: '%s' is not a weight scale for '%s', skipping\n", __func__, name.c_str(), w->name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// the reference computes the router in F32: https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/blob/60d8d70770c6776ff598c94bb586a859a38244f1/inference/model.py#L565
|
||||
if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS || hparams.expert_router_f32) {
|
||||
for (const auto & layer : layers) {
|
||||
prec_policy.set_acc(layer.ffn_gate_inp, GGML_PREC_F32);
|
||||
}
|
||||
}
|
||||
|
||||
// per-tensor activation precision policy
|
||||
prec_policy.load(ml, *this);
|
||||
|
||||
@@ -2575,6 +2618,11 @@ size_t llama_model::size() const {
|
||||
return pimpl->n_bytes;
|
||||
}
|
||||
|
||||
ggml_tensor * llama_model::get_w_scale(const ggml_tensor * w) const {
|
||||
const auto it = w_scales.find(w);
|
||||
return it == w_scales.end() ? nullptr : it->second;
|
||||
}
|
||||
|
||||
size_t llama_model::n_tensors() const {
|
||||
return tensors_by_name.size();
|
||||
}
|
||||
|
||||
+12
-2
@@ -737,8 +737,7 @@ struct llama_meta_device_get_split_state_userdata {
|
||||
struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const struct ggml_tensor * tensor, void * userdata);
|
||||
|
||||
struct llama_prec_policy {
|
||||
// the key is the weight tensor `res->src[0]`, stores the recommended accumulation type of the op (unused for now)
|
||||
// TODO: migrate ad-hoc ggml_prec_set_acc() calls to this container + update apply() to use it
|
||||
// the key is the weight tensor `res->src[0]`, stores the recommended accumulation type of the op
|
||||
std::unordered_map<const ggml_tensor *, ggml_prec> prec_acc;
|
||||
|
||||
// the key is the weight tensor `res->src[0]`, stores the recommended activation precision type
|
||||
@@ -746,6 +745,12 @@ struct llama_prec_policy {
|
||||
|
||||
bool apply(ggml_tensor * res) const;
|
||||
|
||||
void set_acc(const ggml_tensor * w, ggml_prec prec) {
|
||||
if (w) {
|
||||
prec_acc[w] = prec;
|
||||
}
|
||||
}
|
||||
|
||||
void load(llama_model_loader & ml, const llama_model & model);
|
||||
};
|
||||
|
||||
@@ -818,6 +823,11 @@ struct llama_model {
|
||||
|
||||
llama_hadamard hdmd;
|
||||
|
||||
// weight -> "<name>.scale" tensor (e.g. NVFP4 scale2), applied by build_lora_mm and build_lora_mm_id
|
||||
std::unordered_map<const ggml_tensor *, ggml_tensor *> w_scales;
|
||||
|
||||
ggml_tensor * get_w_scale(const ggml_tensor * w) const;
|
||||
|
||||
// see llama_model_set_attn_rot
|
||||
bool attn_rot = true;
|
||||
|
||||
|
||||
@@ -398,10 +398,7 @@ ggml_tensor * llama_model_clover_moe::graph::build_layer_ffn(ggml_tensor * cur,
|
||||
LLM_FFN_SILU, true,
|
||||
hparams.expert_weights_scale,
|
||||
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
|
||||
nullptr, model.layers[il].ffn_gate_up_exps,
|
||||
model.layers[il].ffn_up_exps_s,
|
||||
model.layers[il].ffn_gate_exps_s,
|
||||
model.layers[il].ffn_down_exps_s);
|
||||
nullptr, model.layers[il].ffn_gate_up_exps);
|
||||
cb(moe_out, "ffn_moe_out", il);
|
||||
|
||||
if (model.layers[il].ffn_up_shexp != nullptr) {
|
||||
|
||||
@@ -226,10 +226,7 @@ llama_model_cohere2moe::graph::graph(const llm_graph_params & params) : llm_grap
|
||||
hparams.expert_weights_scale,
|
||||
(llama_expert_gating_func_type) hparams.expert_gating_func,
|
||||
il,
|
||||
nullptr, layer.ffn_gate_up_exps,
|
||||
layer.ffn_up_exps_s,
|
||||
layer.ffn_gate_exps_s,
|
||||
layer.ffn_down_exps_s);
|
||||
nullptr, layer.ffn_gate_up_exps);
|
||||
cb(cur, "ffn_moe_out", il);
|
||||
|
||||
if (layer.ffn_up_shexp) {
|
||||
@@ -371,10 +368,7 @@ llama_model_cohere2moe::graph_mtp::graph_mtp(const llm_graph_params & params) :
|
||||
hparams.expert_weights_scale,
|
||||
(llama_expert_gating_func_type) hparams.expert_gating_func,
|
||||
il,
|
||||
nullptr, layer.ffn_gate_up_exps,
|
||||
layer.ffn_up_exps_s,
|
||||
layer.ffn_gate_exps_s,
|
||||
layer.ffn_down_exps_s);
|
||||
nullptr, layer.ffn_gate_up_exps);
|
||||
cb(cur, "mtp_ffn_moe_out", il);
|
||||
|
||||
if (layer.ffn_up_shexp) {
|
||||
|
||||
@@ -383,10 +383,7 @@ llama_model_deepseek32::graph::graph(const llm_graph_params & params) :
|
||||
(llama_expert_gating_func_type) hparams.expert_gating_func,
|
||||
il,
|
||||
nullptr,
|
||||
model.layers[il].ffn_gate_up_exps,
|
||||
model.layers[il].ffn_up_exps_s,
|
||||
model.layers[il].ffn_gate_exps_s,
|
||||
model.layers[il].ffn_down_exps_s);
|
||||
model.layers[il].ffn_gate_up_exps);
|
||||
cb(moe_out, "ffn_moe_out", il);
|
||||
|
||||
// FFN shared expert
|
||||
@@ -624,10 +621,7 @@ llama_model_deepseek32::graph_mtp::graph_mtp(const llm_graph_params & params)
|
||||
(llama_expert_gating_func_type) hparams.expert_gating_func,
|
||||
il,
|
||||
nullptr,
|
||||
layer.ffn_gate_up_exps,
|
||||
layer.ffn_up_exps_s,
|
||||
layer.ffn_gate_exps_s,
|
||||
layer.ffn_down_exps_s);
|
||||
layer.ffn_gate_up_exps);
|
||||
cb(moe_out, "mtp_ffn_moe_out", il);
|
||||
|
||||
// FFN shared expert
|
||||
|
||||
@@ -1284,9 +1284,6 @@ llama_model_deepseek4::graph::graph(const llm_graph_params & params) :
|
||||
il,
|
||||
nullptr,
|
||||
nullptr,
|
||||
nullptr,
|
||||
nullptr,
|
||||
nullptr,
|
||||
selected_experts);
|
||||
cb(moe_out, "ffn_moe_out", il);
|
||||
|
||||
|
||||
@@ -203,6 +203,8 @@ void llama_model_deepseek41::load_arch_tensors(llama_model_loader & ml) {
|
||||
}
|
||||
|
||||
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
|
||||
// the reference computes the router in F32: https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017/blob/1bae0e1088993b9b82de08f59de5be5a34e42170/inference/model.py#L813
|
||||
prec_policy.set_acc(layer.ffn_gate_inp, GGML_PREC_F32);
|
||||
layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0);
|
||||
layer.ffn_exp_probs_b_vl = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B_VL, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED);
|
||||
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
|
||||
@@ -874,9 +876,7 @@ llama_model_deepseek41::graph::graph(const llm_graph_params & params) :
|
||||
cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
|
||||
cb(cur, "ffn_norm", il);
|
||||
|
||||
// the reference computes the router in F32: https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017/blob/1bae0e1088993b9b82de08f59de5be5a34e42170/inference/model.py#L813
|
||||
ggml_tensor * logits = build_lora_mm(layer.ffn_gate_inp, cur);
|
||||
ggml_prec_set_acc(logits, GGML_PREC_F32);
|
||||
cb(logits, "ffn_moe_logits", il);
|
||||
|
||||
// image (embedding) tokens use the vision router bias, as DeepSeek-V4
|
||||
|
||||
@@ -401,10 +401,7 @@ llama_model_dots3note::graph::graph(const llm_graph_params & params) :
|
||||
(llama_expert_gating_func_type) hparams.expert_gating_func,
|
||||
il,
|
||||
nullptr,
|
||||
model.layers[il].ffn_gate_up_exps,
|
||||
model.layers[il].ffn_up_exps_s,
|
||||
model.layers[il].ffn_gate_exps_s,
|
||||
model.layers[il].ffn_down_exps_s);
|
||||
model.layers[il].ffn_gate_up_exps);
|
||||
cb(moe_out, "ffn_moe_out", il);
|
||||
|
||||
ggml_tensor * ffn_shexp =
|
||||
|
||||
@@ -158,10 +158,7 @@ llama_model_flint::graph::graph(const llm_graph_params & params) :
|
||||
LLM_FFN_SILU, hparams.expert_weights_norm,
|
||||
hparams.expert_weights_scale,
|
||||
(llama_expert_gating_func_type) hparams.expert_gating_func,
|
||||
il, nullptr, nullptr,
|
||||
model.layers[il].ffn_up_exps_s,
|
||||
model.layers[il].ffn_gate_exps_s,
|
||||
model.layers[il].ffn_down_exps_s);
|
||||
il);
|
||||
cb(moe_out, "ffn_moe_out", il);
|
||||
|
||||
{
|
||||
|
||||
@@ -406,10 +406,7 @@ llama_model_gemma4::graph::graph(const llm_graph_params & params) :
|
||||
1.0f,
|
||||
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
|
||||
il, logits,
|
||||
model.layers[il].ffn_gate_up_exps,
|
||||
model.layers[il].ffn_up_exps_s,
|
||||
model.layers[il].ffn_gate_exps_s,
|
||||
model.layers[il].ffn_down_exps_s);
|
||||
model.layers[il].ffn_gate_up_exps);
|
||||
cur_moe = build_norm(cur_moe,
|
||||
model.layers[il].ffn_post_norm_2, nullptr,
|
||||
LLM_NORM_RMS, il);
|
||||
|
||||
@@ -431,10 +431,7 @@ llama_model_glm_dsa::graph::graph(const llm_graph_params & params) :
|
||||
(llama_expert_gating_func_type) hparams.expert_gating_func,
|
||||
il,
|
||||
nullptr,
|
||||
model.layers[il].ffn_gate_up_exps,
|
||||
model.layers[il].ffn_up_exps_s,
|
||||
model.layers[il].ffn_gate_exps_s,
|
||||
model.layers[il].ffn_down_exps_s);
|
||||
model.layers[il].ffn_gate_up_exps);
|
||||
cb(moe_out, "ffn_moe_out", il);
|
||||
|
||||
// FFN shared expert
|
||||
@@ -672,10 +669,7 @@ llama_model_glm_dsa::graph_mtp::graph_mtp(const llm_graph_params & params)
|
||||
(llama_expert_gating_func_type) hparams.expert_gating_func,
|
||||
il,
|
||||
nullptr,
|
||||
layer.ffn_gate_up_exps,
|
||||
layer.ffn_up_exps_s,
|
||||
layer.ffn_gate_exps_s,
|
||||
layer.ffn_down_exps_s);
|
||||
layer.ffn_gate_up_exps);
|
||||
cb(moe_out, "mtp_ffn_moe_out", il);
|
||||
|
||||
// FFN shared expert
|
||||
|
||||
@@ -113,6 +113,11 @@ void llama_model_glm4_moe::load_arch_tensors(llama_model_loader & ml) {
|
||||
layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", i), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED | flags);
|
||||
layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), { n_embd }, TENSOR_NOT_REQUIRED | flags);
|
||||
}
|
||||
|
||||
// numerical issues with F16 accumulators
|
||||
prec_policy.set_acc(layer.wo, GGML_PREC_BF16);
|
||||
prec_policy.set_acc(layer.ffn_down, GGML_PREC_BF16);
|
||||
prec_policy.set_acc(layer.ffn_down_shexp, GGML_PREC_BF16);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -58,6 +58,10 @@ void llama_model_glm4::load_arch_tensors(llama_model_loader &) {
|
||||
layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", i), { n_embd, n_vocab }, flags | TENSOR_NOT_REQUIRED);
|
||||
layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), { n_embd }, flags | TENSOR_NOT_REQUIRED);
|
||||
}
|
||||
|
||||
// numerical issues with F16 accumulators
|
||||
prec_policy.set_acc(layer.wo, GGML_PREC_BF16);
|
||||
prec_policy.set_acc(layer.ffn_down, GGML_PREC_BF16);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -260,11 +260,7 @@ ggml_tensor * llama_model_granite_hybrid::graph::build_layer_ffn(ggml_tensor *
|
||||
LLM_FFN_SILU, true,
|
||||
hparams.expert_weights_scale,
|
||||
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
|
||||
il,
|
||||
nullptr, nullptr,
|
||||
model.layers[il].ffn_up_exps_s,
|
||||
model.layers[il].ffn_gate_exps_s,
|
||||
model.layers[il].ffn_down_exps_s);
|
||||
il);
|
||||
cb(moe_out, "ffn_moe_out", il);
|
||||
|
||||
// For Granite MoE Shared
|
||||
|
||||
@@ -277,10 +277,7 @@ ggml_tensor * llama_model_granite_swa::graph::build_layer_ffn(
|
||||
hparams.expert_weights_scale,
|
||||
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
|
||||
il,
|
||||
nullptr, model.layers[il].ffn_gate_up_exps,
|
||||
model.layers[il].ffn_up_exps_s,
|
||||
model.layers[il].ffn_gate_exps_s,
|
||||
model.layers[il].ffn_down_exps_s);
|
||||
nullptr, model.layers[il].ffn_gate_up_exps);
|
||||
cb(moe_out, "ffn_moe_out", il);
|
||||
|
||||
// For Granite MoE Shared - gate+up kept fused in ffn_up_shexp
|
||||
|
||||
@@ -269,11 +269,7 @@ ggml_tensor * llama_model_granite::graph::build_layer_ffn(
|
||||
LLM_FFN_SILU, true,
|
||||
hparams.expert_weights_scale,
|
||||
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
|
||||
il,
|
||||
nullptr, nullptr,
|
||||
model.layers[il].ffn_up_exps_s,
|
||||
model.layers[il].ffn_gate_exps_s,
|
||||
model.layers[il].ffn_down_exps_s);
|
||||
il);
|
||||
cb(moe_out, "ffn_moe_out", il);
|
||||
|
||||
// For Granite MoE Shared
|
||||
|
||||
@@ -168,10 +168,7 @@ llama_model_hy_v3::graph::graph(const llm_graph_params & params) : llm_graph_con
|
||||
hparams.expert_weights_scale,
|
||||
(llama_expert_gating_func_type) hparams.expert_gating_func,
|
||||
il,
|
||||
nullptr, model.layers[il].ffn_gate_up_exps,
|
||||
model.layers[il].ffn_up_exps_s,
|
||||
model.layers[il].ffn_gate_exps_s,
|
||||
model.layers[il].ffn_down_exps_s);
|
||||
nullptr, model.layers[il].ffn_gate_up_exps);
|
||||
cb(moe_out, "ffn_moe_out", il);
|
||||
|
||||
// shared expert (always active, no gate)
|
||||
@@ -323,10 +320,7 @@ llama_model_hy_v3::graph_mtp::graph_mtp(const llm_graph_params & params)
|
||||
hparams.expert_weights_scale,
|
||||
(llama_expert_gating_func_type) hparams.expert_gating_func,
|
||||
il,
|
||||
nullptr, layer.ffn_gate_up_exps,
|
||||
layer.ffn_up_exps_s,
|
||||
layer.ffn_gate_exps_s,
|
||||
layer.ffn_down_exps_s);
|
||||
nullptr, layer.ffn_gate_up_exps);
|
||||
cb(moe_out, "mtp_ffn_moe_out", il);
|
||||
|
||||
ggml_tensor * sh_out = build_ffn(cur,
|
||||
|
||||
@@ -97,6 +97,7 @@ void llama_model_inkling2::load_arch_tensors(llama_model_loader &) {
|
||||
|
||||
// output
|
||||
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab_padded}, 0);
|
||||
prec_policy.set_acc(output, GGML_PREC_F32);
|
||||
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
|
||||
output_res_query = create_tensor(tn(LLM_TENSOR_OUTPUT_RES_QUERY, "weight"), {n_embd}, 0);
|
||||
|
||||
@@ -406,10 +407,7 @@ ggml_tensor * llama_model_inkling2::graph::build_layer(int il, int ip, ggml_tens
|
||||
LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,
|
||||
il,
|
||||
nullptr,
|
||||
layer.ffn_gate_up_exps,
|
||||
layer.ffn_up_exps_s,
|
||||
layer.ffn_gate_exps_s,
|
||||
layer.ffn_down_exps_s);
|
||||
layer.ffn_gate_up_exps);
|
||||
|
||||
cur = convolve(cur, layer.ffn_post_conv, nullptr, state, history, recr, 2*n_embd_k_gqa + n_embd);
|
||||
cb(cur, "ffn_out", il);
|
||||
@@ -433,7 +431,6 @@ void llama_model_inkling2::graph::build_output(ggml_tensor * cur, const std::vec
|
||||
|
||||
// lm_head
|
||||
cur = build_lora_mm(model.output, cur);
|
||||
ggml_prec_set_acc(cur, GGML_PREC_F32);
|
||||
|
||||
if (hparams.f_final_logit_softcapping != 0.0f) {
|
||||
const float cap = hparams.f_final_logit_softcapping;
|
||||
|
||||
@@ -42,6 +42,10 @@ void llama_model_jais2::load_arch_tensors(llama_model_loader &) {
|
||||
layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, 0);
|
||||
layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);
|
||||
layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, 0);
|
||||
|
||||
// numerical issues with F16 accumulators
|
||||
prec_policy.set_acc(layer.wo, GGML_PREC_BF16);
|
||||
prec_policy.set_acc(layer.ffn_down, GGML_PREC_BF16);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -142,7 +142,7 @@ llama_model_kolibri1::graph::graph(const llm_graph_params & params)
|
||||
(llama_expert_gating_func_type) hparams.expert_gating_func,
|
||||
il,
|
||||
logits,
|
||||
nullptr, nullptr, nullptr, nullptr, selected);
|
||||
nullptr, selected);
|
||||
cb(moe_out, "ffn_moe_out", il);
|
||||
|
||||
ggml_tensor * shared = build_ffn(cur,
|
||||
|
||||
@@ -116,10 +116,5 @@ ggml_tensor * llm_build_lfm_base::build_lfm_moe_ffn(ggml_tensor * cur,
|
||||
LLM_FFN_SILU, true,
|
||||
hparams.expert_weights_scale,
|
||||
static_cast<llama_expert_gating_func_type>(hparams.expert_gating_func),
|
||||
il,
|
||||
nullptr,
|
||||
nullptr,
|
||||
model.layers[il].ffn_up_exps_s,
|
||||
model.layers[il].ffn_gate_exps_s,
|
||||
model.layers[il].ffn_down_exps_s);
|
||||
il);
|
||||
}
|
||||
@@ -198,11 +198,7 @@ llama_model_llama::graph<embed>::graph(const llm_graph_params & params) : llm_gr
|
||||
LLM_FFN_SILU, true,
|
||||
hparams.expert_weights_scale,
|
||||
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
|
||||
il,
|
||||
nullptr, nullptr,
|
||||
model.layers[il].ffn_up_exps_s,
|
||||
model.layers[il].ffn_gate_exps_s,
|
||||
model.layers[il].ffn_down_exps_s);
|
||||
il);
|
||||
cb(cur, "ffn_moe_out", il);
|
||||
}
|
||||
cur = ggml_add(ctx0, cur, ffn_inp);
|
||||
|
||||
+2
-10
@@ -199,11 +199,7 @@ llama_model_mellum::graph<iswa>::graph(const llm_graph_params & params) : llm_gr
|
||||
LLM_FFN_SILU, hparams.expert_weights_norm,
|
||||
hparams.expert_weights_scale,
|
||||
(llama_expert_gating_func_type) hparams.expert_gating_func,
|
||||
il,
|
||||
nullptr, nullptr,
|
||||
model.layers[il].ffn_up_exps_s,
|
||||
model.layers[il].ffn_gate_exps_s,
|
||||
model.layers[il].ffn_down_exps_s);
|
||||
il);
|
||||
cb(moe_out, "ffn_moe_out", il);
|
||||
cur = moe_out;
|
||||
|
||||
@@ -351,11 +347,7 @@ llama_model_mellum::graph_mtp<iswa>::graph_mtp(const llm_graph_params & params)
|
||||
LLM_FFN_SILU, hparams.expert_weights_norm,
|
||||
hparams.expert_weights_scale,
|
||||
(llama_expert_gating_func_type) hparams.expert_gating_func,
|
||||
il,
|
||||
nullptr, nullptr,
|
||||
layer.ffn_up_exps_s,
|
||||
layer.ffn_gate_exps_s,
|
||||
layer.ffn_down_exps_s);
|
||||
il);
|
||||
cb(cur, "mtp_ffn_moe_out", il);
|
||||
|
||||
cur = ggml_add(ctx0, cur, ffn_inp);
|
||||
|
||||
@@ -329,7 +329,7 @@ llama_model_metis16::graph::graph(const llm_graph_params & params) :
|
||||
|
||||
ggml_tensor * moe_out = build_moe_ffn(latent, nullptr, nullptr, nullptr, layer.ffn_down_exps, nullptr,
|
||||
n_exp, n_used, LLM_FFN_SILU, false, 0.0f, LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX_WEIGHT, il_p,
|
||||
logits, layer.ffn_gate_up_exps, nullptr, nullptr, nullptr, selected);
|
||||
logits, layer.ffn_gate_up_exps, selected);
|
||||
ggml_tensor * shexp = build_ffn(latent,
|
||||
layer.ffn_up_shexp, NULL, NULL,
|
||||
layer.ffn_gate_shexp, NULL, NULL,
|
||||
|
||||
@@ -572,10 +572,7 @@ llama_model_minimax_m3::graph::graph(const llm_graph_params & params) : llm_grap
|
||||
LLM_FFN_SWIGLU_OAI_MOE, hparams.expert_weights_norm,
|
||||
hparams.expert_weights_scale,
|
||||
(llama_expert_gating_func_type) hparams.expert_gating_func,
|
||||
il, nullptr, nullptr,
|
||||
model.layers[il].ffn_up_exps_s,
|
||||
model.layers[il].ffn_gate_exps_s,
|
||||
model.layers[il].ffn_down_exps_s);
|
||||
il);
|
||||
cb(moe_out, "ffn_moe_out", il);
|
||||
|
||||
// shared expert (swigluoai)
|
||||
|
||||
@@ -420,10 +420,7 @@ llama_model_mistral_large4::graph::graph(const llm_graph_params & params) :
|
||||
(llama_expert_gating_func_type) hparams.expert_gating_func,
|
||||
il,
|
||||
nullptr,
|
||||
layer.ffn_gate_up_exps,
|
||||
layer.ffn_up_exps_s,
|
||||
layer.ffn_gate_exps_s,
|
||||
layer.ffn_down_exps_s);
|
||||
layer.ffn_gate_up_exps);
|
||||
cb(moe_out, "ffn_moe_out", il);
|
||||
|
||||
ggml_tensor * ffn_shexp = build_ffn(cur,
|
||||
|
||||
@@ -248,11 +248,7 @@ llama_model_mistral3::graph::graph(const llm_graph_params & params) : llm_graph_
|
||||
LLM_FFN_SILU, true,
|
||||
hparams.expert_weights_scale,
|
||||
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
|
||||
il,
|
||||
nullptr, nullptr,
|
||||
model.layers[il].ffn_up_exps_s,
|
||||
model.layers[il].ffn_gate_exps_s,
|
||||
model.layers[il].ffn_down_exps_s);
|
||||
il);
|
||||
cb(cur, "ffn_moe_out", il);
|
||||
}
|
||||
cur = ggml_add(ctx0, cur, ffn_inp);
|
||||
|
||||
@@ -180,7 +180,7 @@ llama_model_muse_spark::graph::graph(const llm_graph_params & params)
|
||||
(llama_expert_gating_func_type) hparams.expert_gating_func,
|
||||
il,
|
||||
router_logits,
|
||||
nullptr, nullptr, nullptr, nullptr, nullptr,
|
||||
nullptr, nullptr,
|
||||
model.layers[il].ffn_post_exp_norm, hparams.f_post_norm_eps);
|
||||
cb(moe_out, "ffn_moe_out", il);
|
||||
|
||||
|
||||
@@ -118,10 +118,7 @@ llama_model_nemotron_h_moe::graph_mtp::graph_mtp(const llm_graph_params & params
|
||||
hparams.expert_weights_scale,
|
||||
LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,
|
||||
il,
|
||||
router_logits, nullptr,
|
||||
layer.ffn_up_exps_s,
|
||||
nullptr, // no gate
|
||||
layer.ffn_down_exps_s);
|
||||
router_logits);
|
||||
cb(moe_out, "mtp_ffn_moe_out", il);
|
||||
|
||||
if (layer.ffn_latent_up) {
|
||||
|
||||
@@ -322,10 +322,7 @@ ggml_tensor * llama_model_nemotron_h::graph::build_ffn_layer(ggml_tensor * cur,
|
||||
hparams.expert_weights_scale,
|
||||
LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,
|
||||
il,
|
||||
router_logits, nullptr,
|
||||
model.layers[il].ffn_up_exps_s,
|
||||
nullptr, // no gate
|
||||
model.layers[il].ffn_down_exps_s);
|
||||
router_logits);
|
||||
cb(moe_out, "ffn_moe_out", il);
|
||||
|
||||
if (model.layers[il].ffn_latent_up) {
|
||||
|
||||
@@ -479,10 +479,7 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_ffn(ggml_tensor * cur, c
|
||||
LLM_FFN_SILU, true,
|
||||
hparams.expert_weights_scale,
|
||||
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
|
||||
nullptr, model.layers[il].ffn_gate_up_exps,
|
||||
model.layers[il].ffn_up_exps_s,
|
||||
model.layers[il].ffn_gate_exps_s,
|
||||
model.layers[il].ffn_down_exps_s);
|
||||
nullptr, model.layers[il].ffn_gate_up_exps);
|
||||
cb(moe_out, "ffn_moe_out", il);
|
||||
|
||||
// Add shared experts if present - following Qwen3Next reference implementation
|
||||
@@ -651,10 +648,7 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llm_graph_params & params)
|
||||
LLM_FFN_SILU, true,
|
||||
hparams.expert_weights_scale,
|
||||
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
|
||||
nullptr, layer.ffn_gate_up_exps,
|
||||
layer.ffn_up_exps_s,
|
||||
layer.ffn_gate_exps_s,
|
||||
layer.ffn_down_exps_s);
|
||||
nullptr, layer.ffn_gate_up_exps);
|
||||
cb(moe_out, "mtp_ffn_moe_out", il);
|
||||
|
||||
if (layer.ffn_up_shexp != nullptr) {
|
||||
|
||||
@@ -136,11 +136,7 @@ llama_model_qwen3moe::graph::graph(const llm_graph_params & params) : llm_graph_
|
||||
LLM_FFN_SILU, true,
|
||||
hparams.expert_weights_scale,
|
||||
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
|
||||
il,
|
||||
nullptr, nullptr,
|
||||
model.layers[il].ffn_up_exps_s,
|
||||
model.layers[il].ffn_gate_exps_s,
|
||||
model.layers[il].ffn_down_exps_s);
|
||||
il);
|
||||
cb(moe_out, "ffn_moe_out", il);
|
||||
cur = moe_out;
|
||||
|
||||
|
||||
@@ -561,10 +561,7 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_ffn(ggml_tensor * cur, c
|
||||
LLM_FFN_SILU, true,
|
||||
hparams.expert_weights_scale,
|
||||
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
|
||||
nullptr, model.layers[il].ffn_gate_up_exps,
|
||||
model.layers[il].ffn_up_exps_s,
|
||||
model.layers[il].ffn_gate_exps_s,
|
||||
model.layers[il].ffn_down_exps_s);
|
||||
nullptr, model.layers[il].ffn_gate_up_exps);
|
||||
cb(moe_out, "ffn_moe_out", il);
|
||||
|
||||
// Add shared experts if present - following Qwen3Next reference implementation
|
||||
@@ -748,10 +745,7 @@ llama_model_qwen3next::graph_mtp::graph_mtp(const llm_graph_params & params)
|
||||
LLM_FFN_SILU, true,
|
||||
hparams.expert_weights_scale,
|
||||
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
|
||||
nullptr, layer.ffn_gate_up_exps,
|
||||
layer.ffn_up_exps_s,
|
||||
layer.ffn_gate_exps_s,
|
||||
layer.ffn_down_exps_s);
|
||||
nullptr, layer.ffn_gate_up_exps);
|
||||
cb(moe_out, "mtp_ffn_moe_out", il);
|
||||
|
||||
if (layer.ffn_up_shexp != nullptr) {
|
||||
|
||||
@@ -1230,10 +1230,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_ffn(ggml_tensor * cur, co
|
||||
LLM_FFN_SILU, true,
|
||||
hparams.expert_weights_scale,
|
||||
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
|
||||
nullptr, model.layers[il].ffn_gate_up_exps,
|
||||
model.layers[il].ffn_up_exps_s,
|
||||
model.layers[il].ffn_gate_exps_s,
|
||||
model.layers[il].ffn_down_exps_s);
|
||||
nullptr, model.layers[il].ffn_gate_up_exps);
|
||||
cb(moe_out, "ffn_moe_out", il);
|
||||
|
||||
// shared experts, as in the Qwen3Next reference
|
||||
|
||||
@@ -277,9 +277,6 @@ ggml_tensor * llama_model_zaya1::graph::build_router_moe(const llama_model & mod
|
||||
il,
|
||||
logits,
|
||||
layer.ffn_gate_up_exps,
|
||||
nullptr,
|
||||
nullptr,
|
||||
nullptr,
|
||||
expert);
|
||||
|
||||
return ggml_mul(ctx0, moe, keep);
|
||||
|
||||
@@ -41,6 +41,7 @@ cohere2 ,0 ,any ,NORM+MUL , 3
|
||||
cohere2moe ,1 ,any ,ADD+ADD , 2
|
||||
cohere2moe ,1 ,any ,MUL+ADD , 2
|
||||
cohere2moe ,1 ,any ,RMS_NORM+MUL , 3
|
||||
command-r ,0 ,any ,ADD+ADD , 2
|
||||
command-r ,0 ,any ,NORM+MUL , 3
|
||||
dbrx ,0 ,any ,MUL+ADD , 4
|
||||
dbrx ,0 ,any ,NORM+MUL , 5
|
||||
@@ -207,6 +208,7 @@ llama ,0 ,any ,MUL+ADD , 4
|
||||
llama ,0 ,any ,RMS_NORM+MUL , 5
|
||||
llama ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
|
||||
llama4 ,0 ,any ,ADD+ADD , 2
|
||||
llama4 ,0 ,any ,MUL+ADD , 4
|
||||
llama4 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
maincoder ,0 ,any ,RMS_NORM+MUL , 9
|
||||
maion-coder ,0 ,any ,MUL+ADD , 4
|
||||
@@ -257,8 +259,7 @@ moshi ,0 ,any ,RMS_NORM+MUL , 5
|
||||
mpt ,0 ,any ,NORM+MUL+ADD , 5
|
||||
muse-glimmer ,0 ,any ,RMS_NORM+MUL , 10
|
||||
muse-glimmer ,0 ,any ,RMS_NORM+MUL+ADD , 3
|
||||
muse-spark ,0 ,prefill ,MUL+ADD , 2
|
||||
muse-spark ,0 ,decode ,MUL+ADD , 4
|
||||
muse-spark ,0 ,any ,MUL+ADD , 4
|
||||
muse-spark ,0 ,any ,RMS_NORM+MUL , 11
|
||||
muse-spark ,0 ,any ,RMS_NORM+MUL+ADD , 3
|
||||
muse-spark ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
|
||||
|
||||
|
Reference in new issue
Block a user