model: apply .scale tensors automatically + fix correctness (#32669)

* model: apply .scale tensors automatically

* rm build_moe_ffn "_s" overload

* use llama_prec_policy

* fix ci

* build_lora_mm_id: rm w_s
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-11-21 16:30:01 +01:00
1 parent e50c254a22
commit 19bc41dbb4
39 files changed
+164 -290

No files matched your search

+24 -18
View File
@@ -2154,8 +2154,28 @@ static enum ggml_status ggml_backend_meta_graph_compute(ggml_backend_t backend,
}
};
skip_unrelated();
if (id + 1 >= cgraph->n_nodes) {
// Chain of MULs with MIRRORED src[1], returns false at the end of the graph
auto absorb_muls = [&]() -> bool {
while (true) {
skip_unrelated();
if (id + 1 >= cgraph->n_nodes) {
return false;
}
ggml_tensor * next = cgraph->nodes[id+1];
if (next->op == GGML_OP_MUL && next->src[0] == node &&
ggml_backend_meta_get_split_state(next->src[1], false).axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED) {
node = next;
id++;
idr = id;
n_used = ggml_node_get_use_count(cgraph, id);
} else {
return true;
}
}
};
// MULs before the PARTIAL bias (e.g. a per-expert weight scale): sum_j(s*p_j + b/n) == s*sum_j(p_j) + b
if (!absorb_muls()) {
return idr;
}
{
@@ -2169,22 +2189,8 @@ static enum ggml_status ggml_backend_meta_graph_compute(ggml_backend_t backend,
n_used = ggml_node_get_use_count(cgraph, id);
}
}
// Chain of MULs with MIRRORED src[1]
while (true) {
skip_unrelated();
if (id + 1 >= cgraph->n_nodes) {
return idr;
}
ggml_tensor * next = cgraph->nodes[id+1];
if (next->op == GGML_OP_MUL && next->src[0] == node &&
ggml_backend_meta_get_split_state(next->src[1], false).axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED) {
node = next;
id++;
idr = id;
n_used = ggml_node_get_use_count(cgraph, id);
} else {
break;
}
if (!absorb_muls()) {
return idr;
}
if (n_used != node->ne[1] || id + 2*n_used-1 >= cgraph->n_nodes) {
+22 -112
View File
@@ -1810,16 +1810,21 @@ ggml_tensor * llm_graph_context::build_lora_mm(
ggml_tensor * w,
ggml_tensor * cur,
ggml_tensor * w_s) const {
if (!w_s) {
w_s = model.get_w_scale(w);
}
// a Hadamard-folded weight reads the transformed input; a LoRA delta lives in the primal basis and reads `cur`
ggml_tensor * cur_mm = build_hadamard_input(w, cur);
ggml_tensor * res = ggml_mul_mat(ctx0, w, cur_mm);
prec_policy.apply(res);
if (w->type == GGML_TYPE_NVFP4) {
ggml_prec_set_acc(res, GGML_PREC_BF16);
}
prec_policy.apply(res);
if (w_s) {
res = ggml_mul(ctx0, res, w_s);
}
@@ -1886,9 +1891,10 @@ ggml_tensor * llm_graph_context::build_lora_mm_id(
ggml_tensor * w, // ggml_tensor * as
ggml_tensor * cur, // ggml_tensor * b
ggml_tensor * ids,
ggml_tensor * w_s,
ggml_tensor * slots,
bool cached) const {
ggml_tensor * w_s = model.get_w_scale(w);
// as in build_lora_mm: the folded weight reads the transformed input, the LoRA delta reads `cur`
ggml_tensor * cur_mm = build_hadamard_input(w, cur);
// the experts in the MoE cache are selected by their slots
@@ -1896,12 +1902,13 @@ ggml_tensor * llm_graph_context::build_lora_mm_id(
ggml_mul_mat_id(ctx0, w, cur_mm, ids) :
ggml_mul_mat_id(ctx0, cached ? moe_cache->get_experts(w) : w, cur_mm, slots);
prec_policy.apply(res);
if (w->type == GGML_TYPE_NVFP4) {
ggml_prec_set_acc(res, GGML_PREC_BF16);
}
// after the NVFP4 default, so that the policy can request a higher precision
prec_policy.apply(res);
if (w_s) {
const int64_t n_expert = w_s->ne[0];
const int64_t n_tokens = cur->ne[2];
@@ -2115,34 +2122,7 @@ ggml_tensor * llm_graph_context::build_ffn(
int il) const {
const llm_arch arch = model.arch;
// NVFP4 support is currently restricted to
// 1) LORA absence (*_s would be applied after LORA residual, which is incorrect)
// 2) bias absense (*_s would be applied after bias addition, which is incorrect)
// TODO: disambiguate LLM-architectural scales (which use *_s) from NVFP4 scale_2 (which also uses *_s currently)
auto has_lora = [this](ggml_tensor * w) {
if (!w) {
return false;
}
for (const auto & lora : *loras) {
if (lora.first->get_weight(w) != nullptr) {
return true;
}
}
return false;
};
auto is_scaled_low_precision = [](ggml_tensor * w) {
return w && (w->type == GGML_TYPE_NVFP4 || w->type == GGML_TYPE_F8_E4M3);
};
GGML_ASSERT(!up_s || !up_b || !is_scaled_low_precision(up));
GGML_ASSERT(!gate_s || !gate_b || !is_scaled_low_precision(gate));
GGML_ASSERT(!down_s || !down_b || !is_scaled_low_precision(down));
GGML_ASSERT(!up_s || !is_scaled_low_precision(up) || !has_lora(up));
GGML_ASSERT(!gate_s || !is_scaled_low_precision(gate) || !has_lora(gate));
GGML_ASSERT(!down_s || !is_scaled_low_precision(down) || !has_lora(down));
ggml_tensor * tmp = up ? build_lora_mm(up, cur) : cur;
ggml_tensor * tmp = up ? build_lora_mm(up, cur, up_s) : cur;
cb(tmp, "ffn_up", il);
if (up_b) {
@@ -2150,21 +2130,16 @@ ggml_tensor * llm_graph_context::build_ffn(
cb(tmp, "ffn_up_b", il);
}
if (up_s) {
tmp = ggml_mul(ctx0, tmp, up_s);
cb(tmp, "ffn_up_s", il);
}
if (gate) {
switch (type_gate) {
case LLM_FFN_SEQ:
{
cur = build_lora_mm(gate, tmp);
cur = build_lora_mm(gate, tmp, gate_s);
cb(cur, "ffn_gate", il);
} break;
case LLM_FFN_PAR:
{
cur = build_lora_mm(gate, cur);
cur = build_lora_mm(gate, cur, gate_s);
cb(cur, "ffn_gate", il);
} break;
}
@@ -2174,11 +2149,6 @@ ggml_tensor * llm_graph_context::build_ffn(
cb(cur, "ffn_gate_b", il);
}
if (gate_s) {
cur = ggml_mul(ctx0, cur, gate_s);
cb(cur, "ffn_gate_s", il);
}
} else {
cur = tmp;
}
@@ -2300,11 +2270,7 @@ ggml_tensor * llm_graph_context::build_ffn(
}
if (down) {
cur = build_lora_mm(down, cur);
if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) {
// GLM4, GLM4_MOE, and JAIS2 have numerical issues with F16 accumulators
ggml_prec_set_acc(cur, GGML_PREC_BF16);
}
cur = build_lora_mm(down, cur, down_s);
}
if (down_b) {
@@ -2315,11 +2281,6 @@ ggml_tensor * llm_graph_context::build_ffn(
cur = ggml_add(ctx0, cur, down_b);
}
if (down_s) {
cur = ggml_mul(ctx0, cur, down_s);
cb(cur, "ffn_down_s", il);
}
return cur;
}
@@ -2339,9 +2300,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
int il,
ggml_tensor * probs_in,
ggml_tensor * gate_up_exps,
ggml_tensor * up_exps_s,
ggml_tensor * gate_exps_s,
ggml_tensor * down_exps_s,
ggml_tensor * selected_experts_in,
ggml_tensor * post_exp_norm,
float post_exp_norm_eps) const {
@@ -2362,9 +2320,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
probs_in,
gate_up_exps,
/* gate_up_exps_b */ nullptr,
up_exps_s,
gate_exps_s,
down_exps_s,
selected_experts_in,
post_exp_norm,
post_exp_norm_eps
@@ -2392,9 +2347,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
ggml_tensor * probs_in,
ggml_tensor * gate_up_exps,
ggml_tensor * gate_up_exps_b,
ggml_tensor * up_exps_s,
ggml_tensor * gate_exps_s,
ggml_tensor * down_exps_s,
ggml_tensor * selected_experts_in,
ggml_tensor * post_exp_norm,
float post_exp_norm_eps) const {
@@ -2407,10 +2359,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
if (probs_in == nullptr) {
logits = build_lora_mm(gate_inp, cur); // [n_expert, n_tokens]
if (gating_op == LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS || hparams.expert_router_f32) {
// the reference computes the router in F32: https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/blob/60d8d70770c6776ff598c94bb586a859a38244f1/inference/model.py#L565
ggml_prec_set_acc(logits, GGML_PREC_F32);
}
cb(logits, "ffn_moe_logits", il);
} else {
logits = probs_in;
@@ -2583,13 +2531,9 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
if (gate_up_exps) {
// merged gate_up path: one mul_mat_id, then split into gate and up views
ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, cur, ids, up_exps_s, mm_ids, cached); // [n_ff*2, n_expert_used, n_tokens]
ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, cur, ids, mm_ids, cached); // [n_ff*2, n_expert_used, n_tokens]
cb(gate_up, "ffn_moe_gate_up", il);
if (up_exps_s) {
cb(gate_up, "ffn_moe_gate_up_scaled", il);
}
if (gate_up_exps_b) {
gate_up = ggml_add_id(ctx0, gate_up, gate_up_exps_b, ids);
cb(gate_up, "ffn_moe_gate_up_biased", il);
@@ -2602,29 +2546,21 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
cb(up, "ffn_moe_up", il);
} else {
// separate gate and up path
up = build_lora_mm_id(up_exps, cur, ids, up_exps_s, mm_ids, cached); // [n_ff, n_expert_used, n_tokens]
up = build_lora_mm_id(up_exps, cur, ids, mm_ids, cached); // [n_ff, n_expert_used, n_tokens]
cb(up, "ffn_moe_up", il);
if (up_exps_s) {
cb(up, "ffn_moe_up_scaled", il);
}
if (up_exps_b) {
up = ggml_add_id(ctx0, up, up_exps_b, ids);
cb(up, "ffn_moe_up_biased", il);
}
if (gate_exps) {
cur = build_lora_mm_id(gate_exps, cur, ids, gate_exps_s, mm_ids, cached); // [n_ff, n_expert_used, n_tokens]
cur = build_lora_mm_id(gate_exps, cur, ids, mm_ids, cached); // [n_ff, n_expert_used, n_tokens]
cb(cur, "ffn_moe_gate", il);
} else {
cur = up;
}
if (gate_exps_s) {
cb(cur, "ffn_moe_gate_scaled", il);
}
if (gate_exps_b) {
cur = ggml_add_id(ctx0, cur, gate_exps_b, ids);
cb(cur, "ffn_moe_gate_biased", il);
@@ -2727,7 +2663,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
cur = amax_src1.res;
}
ggml_tensor * experts = build_lora_mm_id(down_exps, cur, ids, down_exps_s, mm_ids, cached); // [n_embd, n_expert_used, n_tokens]
ggml_tensor * experts = build_lora_mm_id(down_exps, cur, ids, mm_ids, cached); // [n_embd, n_expert_used, n_tokens]
//ggml_prec_set_src(experts, GGML_PREC_BF16, 1); // (see TODO above)
cb(experts, "ffn_moe_down", il);
@@ -2745,10 +2681,6 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
return (*static_cast<decltype(build_experts) *>(user_data))(cur, ids, mm_ids, cached);
}, &build_experts);
if (down_exps_s) {
cb(experts, "ffn_moe_down_scaled", il);
}
if (down_exps_b) {
experts = ggml_add_id(ctx0, experts, down_exps_b, selected_experts);
cb(experts, "ffn_moe_down_biased", il);
@@ -3485,8 +3417,6 @@ ggml_tensor * llm_graph_context::build_attn(
ggml_tensor * v_mla, // TODO: remove
float kq_scale,
int il) const {
const llm_arch arch = model.arch;
GGML_ASSERT(v_mla == nullptr);
if (inp->self_k_rot[il]) {
@@ -3543,16 +3473,7 @@ ggml_tensor * llm_graph_context::build_attn(
}
if (wo) {
if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) {
// GLM4, GLM4_MOE, and JAIS2 have numerical issues with F16 accumulators
cur = build_lora_mm(wo, cur);
ggml_prec_set_acc(cur, GGML_PREC_BF16);
if (wo_s) {
cur = ggml_mul(ctx0, cur, wo_s);
}
} else {
cur = build_lora_mm(wo, cur, wo_s);
}
cur = build_lora_mm(wo, cur, wo_s);
}
if (wo_b) {
@@ -3603,8 +3524,6 @@ ggml_tensor * llm_graph_context::build_attn(
ggml_tensor * v_mla,
float kq_scale,
int il) const {
const llm_arch arch = model.arch;
// these nodes are added to the graph together so that they are not reordered
// by doing so, the number of splits in the graph is reduced
// expand k later to enable rope fusion which directly writes into k-v cache
@@ -3632,16 +3551,7 @@ ggml_tensor * llm_graph_context::build_attn(
cb(cur, "kqv_out", il);
if (wo) {
if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE) {
// GLM4 and GLM4_MOE have numerical issues with F16 accumulators
cur = build_lora_mm(wo, cur);
ggml_prec_set_acc(cur, GGML_PREC_BF16);
if (wo_s) {
cur = ggml_mul(ctx0, cur, wo_s);
}
} else {
cur = build_lora_mm(wo, cur, wo_s);
}
cur = build_lora_mm(wo, cur, wo_s);
}
if (wo_b) {
+1 -7
View File
@@ -1179,6 +1179,7 @@ struct llm_graph_context {
ggml_tensor * cur) const;
// do mat_mul, while optionally apply lora and per-tensor scale
// w_s defaults to the weight scale of w
ggml_tensor * build_lora_mm(
ggml_tensor * w,
ggml_tensor * cur,
@@ -1190,7 +1191,6 @@ struct llm_graph_context {
ggml_tensor * w, // ggml_tensor * as
ggml_tensor * cur, // ggml_tensor * b
ggml_tensor * ids,
ggml_tensor * w_s = nullptr,
ggml_tensor * slots = nullptr,
bool cached = true) const;
@@ -1263,9 +1263,6 @@ struct llm_graph_context {
int il,
ggml_tensor * probs_in = nullptr,
ggml_tensor * gate_up_exps = nullptr,
ggml_tensor * up_exps_s = nullptr,
ggml_tensor * gate_exps_s = nullptr,
ggml_tensor * down_exps_s = nullptr,
ggml_tensor * selected_experts_in = nullptr,
ggml_tensor * post_exp_norm = nullptr,
float post_exp_norm_eps = 1e-5f) const;
@@ -1291,9 +1288,6 @@ struct llm_graph_context {
ggml_tensor * probs_in = nullptr,
ggml_tensor * gate_up_exps = nullptr,
ggml_tensor * gate_up_exps_b = nullptr,
ggml_tensor * up_exps_s = nullptr,
ggml_tensor * gate_exps_s = nullptr,
ggml_tensor * down_exps_s = nullptr,
ggml_tensor * selected_experts_in = nullptr,
ggml_tensor * post_exp_norm = nullptr,
float post_exp_norm_eps = 1e-5f) const;
+52 -4
View File
@@ -1410,12 +1410,19 @@ bool llama_prec_policy::apply(ggml_tensor * res) const {
return false;
}
const auto it = prec_src1.find(res->src[0]);
if (it == prec_src1.end()) {
return false;
bool applied = false;
const auto it_acc = prec_acc.find(res->src[0]);
if (it_acc != prec_acc.end()) {
applied |= ggml_prec_set_acc(res, it_acc->second);
}
return ggml_prec_set_src(res, it->second, 1);
const auto it_src1 = prec_src1.find(res->src[0]);
if (it_src1 != prec_src1.end()) {
applied |= ggml_prec_set_src(res, it_src1->second, 1);
}
return applied;
}
void llama_prec_policy::load(llama_model_loader & ml, const llama_model & model) {
@@ -2044,6 +2051,42 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
}
}
// map each weight to its "<name>.scale" tensor: shape {1} for a 2D weight, {n_expert} for a 3D weight
{
auto ends_with = [](const std::string & str, const std::string & suffix) {
return str.size() >= suffix.size() && str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0;
};
std::unordered_map<std::string, ggml_tensor *> weights;
for (const auto & [name, cur] : tensors_by_name) {
if (ends_with(name, ".weight")) {
weights.emplace(name.substr(0, name.size() - 7), cur);
}
}
for (const auto & [name, s] : tensors_by_name) {
if (!ends_with(name, ".scale")) {
continue;
}
const auto it = weights.find(name.substr(0, name.size() - 6));
if (it == weights.end()) {
continue;
}
ggml_tensor * w = it->second;
const bool ok = w->ne[3] == 1 && ggml_nelements(s) == w->ne[2] && s->ne[0] == w->ne[2];
if (ok) {
w_scales.emplace(w, s);
} else {
LLAMA_LOG_DEBUG("%s: '%s' is not a weight scale for '%s', skipping\n", __func__, name.c_str(), w->name);
}
}
}
// the reference computes the router in F32: https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/blob/60d8d70770c6776ff598c94bb586a859a38244f1/inference/model.py#L565
if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS || hparams.expert_router_f32) {
for (const auto & layer : layers) {
prec_policy.set_acc(layer.ffn_gate_inp, GGML_PREC_F32);
}
}
// per-tensor activation precision policy
prec_policy.load(ml, *this);
@@ -2575,6 +2618,11 @@ size_t llama_model::size() const {
return pimpl->n_bytes;
}
ggml_tensor * llama_model::get_w_scale(const ggml_tensor * w) const {
const auto it = w_scales.find(w);
return it == w_scales.end() ? nullptr : it->second;
}
size_t llama_model::n_tensors() const {
return tensors_by_name.size();
}
+12 -2
View File
@@ -737,8 +737,7 @@ struct llama_meta_device_get_split_state_userdata {
struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const struct ggml_tensor * tensor, void * userdata);
struct llama_prec_policy {
// the key is the weight tensor `res->src[0]`, stores the recommended accumulation type of the op (unused for now)
// TODO: migrate ad-hoc ggml_prec_set_acc() calls to this container + update apply() to use it
// the key is the weight tensor `res->src[0]`, stores the recommended accumulation type of the op
std::unordered_map<const ggml_tensor *, ggml_prec> prec_acc;
// the key is the weight tensor `res->src[0]`, stores the recommended activation precision type
@@ -746,6 +745,12 @@ struct llama_prec_policy {
bool apply(ggml_tensor * res) const;
void set_acc(const ggml_tensor * w, ggml_prec prec) {
if (w) {
prec_acc[w] = prec;
}
}
void load(llama_model_loader & ml, const llama_model & model);
};
@@ -818,6 +823,11 @@ struct llama_model {
llama_hadamard hdmd;
// weight -> "<name>.scale" tensor (e.g. NVFP4 scale2), applied by build_lora_mm and build_lora_mm_id
std::unordered_map<const ggml_tensor *, ggml_tensor *> w_scales;
ggml_tensor * get_w_scale(const ggml_tensor * w) const;
// see llama_model_set_attn_rot
bool attn_rot = true;
+1 -4
View File
@@ -398,10 +398,7 @@ ggml_tensor * llama_model_clover_moe::graph::build_layer_ffn(ggml_tensor * cur,
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
nullptr, model.layers[il].ffn_gate_up_exps,
model.layers[il].ffn_up_exps_s,
model.layers[il].ffn_gate_exps_s,
model.layers[il].ffn_down_exps_s);
nullptr, model.layers[il].ffn_gate_up_exps);
cb(moe_out, "ffn_moe_out", il);
if (model.layers[il].ffn_up_shexp != nullptr) {
+2 -8
View File
@@ -226,10 +226,7 @@ llama_model_cohere2moe::graph::graph(const llm_graph_params & params) : llm_grap
hparams.expert_weights_scale,
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
nullptr, layer.ffn_gate_up_exps,
layer.ffn_up_exps_s,
layer.ffn_gate_exps_s,
layer.ffn_down_exps_s);
nullptr, layer.ffn_gate_up_exps);
cb(cur, "ffn_moe_out", il);
if (layer.ffn_up_shexp) {
@@ -371,10 +368,7 @@ llama_model_cohere2moe::graph_mtp::graph_mtp(const llm_graph_params & params) :
hparams.expert_weights_scale,
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
nullptr, layer.ffn_gate_up_exps,
layer.ffn_up_exps_s,
layer.ffn_gate_exps_s,
layer.ffn_down_exps_s);
nullptr, layer.ffn_gate_up_exps);
cb(cur, "mtp_ffn_moe_out", il);
if (layer.ffn_up_shexp) {
+2 -8
View File
@@ -383,10 +383,7 @@ llama_model_deepseek32::graph::graph(const llm_graph_params & params) :
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
nullptr,
model.layers[il].ffn_gate_up_exps,
model.layers[il].ffn_up_exps_s,
model.layers[il].ffn_gate_exps_s,
model.layers[il].ffn_down_exps_s);
model.layers[il].ffn_gate_up_exps);
cb(moe_out, "ffn_moe_out", il);
// FFN shared expert
@@ -624,10 +621,7 @@ llama_model_deepseek32::graph_mtp::graph_mtp(const llm_graph_params & params)
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
nullptr,
layer.ffn_gate_up_exps,
layer.ffn_up_exps_s,
layer.ffn_gate_exps_s,
layer.ffn_down_exps_s);
layer.ffn_gate_up_exps);
cb(moe_out, "mtp_ffn_moe_out", il);
// FFN shared expert
-3
View File
@@ -1284,9 +1284,6 @@ llama_model_deepseek4::graph::graph(const llm_graph_params & params) :
il,
nullptr,
nullptr,
nullptr,
nullptr,
nullptr,
selected_experts);
cb(moe_out, "ffn_moe_out", il);
+2 -2
View File
@@ -203,6 +203,8 @@ void llama_model_deepseek41::load_arch_tensors(llama_model_loader & ml) {
}
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
// the reference computes the router in F32: https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017/blob/1bae0e1088993b9b82de08f59de5be5a34e42170/inference/model.py#L813
prec_policy.set_acc(layer.ffn_gate_inp, GGML_PREC_F32);
layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0);
layer.ffn_exp_probs_b_vl = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B_VL, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
@@ -874,9 +876,7 @@ llama_model_deepseek41::graph::graph(const llm_graph_params & params) :
cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
cb(cur, "ffn_norm", il);
// the reference computes the router in F32: https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017/blob/1bae0e1088993b9b82de08f59de5be5a34e42170/inference/model.py#L813
ggml_tensor * logits = build_lora_mm(layer.ffn_gate_inp, cur);
ggml_prec_set_acc(logits, GGML_PREC_F32);
cb(logits, "ffn_moe_logits", il);
// image (embedding) tokens use the vision router bias, as DeepSeek-V4
+1 -4
View File
@@ -401,10 +401,7 @@ llama_model_dots3note::graph::graph(const llm_graph_params & params) :
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
nullptr,
model.layers[il].ffn_gate_up_exps,
model.layers[il].ffn_up_exps_s,
model.layers[il].ffn_gate_exps_s,
model.layers[il].ffn_down_exps_s);
model.layers[il].ffn_gate_up_exps);
cb(moe_out, "ffn_moe_out", il);
ggml_tensor * ffn_shexp =
+1 -4
View File
@@ -158,10 +158,7 @@ llama_model_flint::graph::graph(const llm_graph_params & params) :
LLM_FFN_SILU, hparams.expert_weights_norm,
hparams.expert_weights_scale,
(llama_expert_gating_func_type) hparams.expert_gating_func,
il, nullptr, nullptr,
model.layers[il].ffn_up_exps_s,
model.layers[il].ffn_gate_exps_s,
model.layers[il].ffn_down_exps_s);
il);
cb(moe_out, "ffn_moe_out", il);
{
+1 -4
View File
@@ -406,10 +406,7 @@ llama_model_gemma4::graph::graph(const llm_graph_params & params) :
1.0f,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
il, logits,
model.layers[il].ffn_gate_up_exps,
model.layers[il].ffn_up_exps_s,
model.layers[il].ffn_gate_exps_s,
model.layers[il].ffn_down_exps_s);
model.layers[il].ffn_gate_up_exps);
cur_moe = build_norm(cur_moe,
model.layers[il].ffn_post_norm_2, nullptr,
LLM_NORM_RMS, il);
+2 -8
View File
@@ -431,10 +431,7 @@ llama_model_glm_dsa::graph::graph(const llm_graph_params & params) :
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
nullptr,
model.layers[il].ffn_gate_up_exps,
model.layers[il].ffn_up_exps_s,
model.layers[il].ffn_gate_exps_s,
model.layers[il].ffn_down_exps_s);
model.layers[il].ffn_gate_up_exps);
cb(moe_out, "ffn_moe_out", il);
// FFN shared expert
@@ -672,10 +669,7 @@ llama_model_glm_dsa::graph_mtp::graph_mtp(const llm_graph_params & params)
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
nullptr,
layer.ffn_gate_up_exps,
layer.ffn_up_exps_s,
layer.ffn_gate_exps_s,
layer.ffn_down_exps_s);
layer.ffn_gate_up_exps);
cb(moe_out, "mtp_ffn_moe_out", il);
// FFN shared expert
+5
View File
@@ -113,6 +113,11 @@ void llama_model_glm4_moe::load_arch_tensors(llama_model_loader & ml) {
layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", i), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED | flags);
layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), { n_embd }, TENSOR_NOT_REQUIRED | flags);
}
// numerical issues with F16 accumulators
prec_policy.set_acc(layer.wo, GGML_PREC_BF16);
prec_policy.set_acc(layer.ffn_down, GGML_PREC_BF16);
prec_policy.set_acc(layer.ffn_down_shexp, GGML_PREC_BF16);
}
}
+4
View File
@@ -58,6 +58,10 @@ void llama_model_glm4::load_arch_tensors(llama_model_loader &) {
layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", i), { n_embd, n_vocab }, flags | TENSOR_NOT_REQUIRED);
layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), { n_embd }, flags | TENSOR_NOT_REQUIRED);
}
// numerical issues with F16 accumulators
prec_policy.set_acc(layer.wo, GGML_PREC_BF16);
prec_policy.set_acc(layer.ffn_down, GGML_PREC_BF16);
}
}
+1 -5
View File
@@ -260,11 +260,7 @@ ggml_tensor * llama_model_granite_hybrid::graph::build_layer_ffn(ggml_tensor *
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
il,
nullptr, nullptr,
model.layers[il].ffn_up_exps_s,
model.layers[il].ffn_gate_exps_s,
model.layers[il].ffn_down_exps_s);
il);
cb(moe_out, "ffn_moe_out", il);
// For Granite MoE Shared
+1 -4
View File
@@ -277,10 +277,7 @@ ggml_tensor * llama_model_granite_swa::graph::build_layer_ffn(
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
il,
nullptr, model.layers[il].ffn_gate_up_exps,
model.layers[il].ffn_up_exps_s,
model.layers[il].ffn_gate_exps_s,
model.layers[il].ffn_down_exps_s);
nullptr, model.layers[il].ffn_gate_up_exps);
cb(moe_out, "ffn_moe_out", il);
// For Granite MoE Shared - gate+up kept fused in ffn_up_shexp
+1 -5
View File
@@ -269,11 +269,7 @@ ggml_tensor * llama_model_granite::graph::build_layer_ffn(
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
il,
nullptr, nullptr,
model.layers[il].ffn_up_exps_s,
model.layers[il].ffn_gate_exps_s,
model.layers[il].ffn_down_exps_s);
il);
cb(moe_out, "ffn_moe_out", il);
// For Granite MoE Shared
+2 -8
View File
@@ -168,10 +168,7 @@ llama_model_hy_v3::graph::graph(const llm_graph_params & params) : llm_graph_con
hparams.expert_weights_scale,
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
nullptr, model.layers[il].ffn_gate_up_exps,
model.layers[il].ffn_up_exps_s,
model.layers[il].ffn_gate_exps_s,
model.layers[il].ffn_down_exps_s);
nullptr, model.layers[il].ffn_gate_up_exps);
cb(moe_out, "ffn_moe_out", il);
// shared expert (always active, no gate)
@@ -323,10 +320,7 @@ llama_model_hy_v3::graph_mtp::graph_mtp(const llm_graph_params & params)
hparams.expert_weights_scale,
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
nullptr, layer.ffn_gate_up_exps,
layer.ffn_up_exps_s,
layer.ffn_gate_exps_s,
layer.ffn_down_exps_s);
nullptr, layer.ffn_gate_up_exps);
cb(moe_out, "mtp_ffn_moe_out", il);
ggml_tensor * sh_out = build_ffn(cur,
+2 -5
View File
@@ -97,6 +97,7 @@ void llama_model_inkling2::load_arch_tensors(llama_model_loader &) {
// output
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab_padded}, 0);
prec_policy.set_acc(output, GGML_PREC_F32);
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
output_res_query = create_tensor(tn(LLM_TENSOR_OUTPUT_RES_QUERY, "weight"), {n_embd}, 0);
@@ -406,10 +407,7 @@ ggml_tensor * llama_model_inkling2::graph::build_layer(int il, int ip, ggml_tens
LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,
il,
nullptr,
layer.ffn_gate_up_exps,
layer.ffn_up_exps_s,
layer.ffn_gate_exps_s,
layer.ffn_down_exps_s);
layer.ffn_gate_up_exps);
cur = convolve(cur, layer.ffn_post_conv, nullptr, state, history, recr, 2*n_embd_k_gqa + n_embd);
cb(cur, "ffn_out", il);
@@ -433,7 +431,6 @@ void llama_model_inkling2::graph::build_output(ggml_tensor * cur, const std::vec
// lm_head
cur = build_lora_mm(model.output, cur);
ggml_prec_set_acc(cur, GGML_PREC_F32);
if (hparams.f_final_logit_softcapping != 0.0f) {
const float cap = hparams.f_final_logit_softcapping;
+4
View File
@@ -42,6 +42,10 @@ void llama_model_jais2::load_arch_tensors(llama_model_loader &) {
layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, 0);
layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);
layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, 0);
// numerical issues with F16 accumulators
prec_policy.set_acc(layer.wo, GGML_PREC_BF16);
prec_policy.set_acc(layer.ffn_down, GGML_PREC_BF16);
}
}
+1 -1
View File
@@ -142,7 +142,7 @@ llama_model_kolibri1::graph::graph(const llm_graph_params & params)
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
logits,
nullptr, nullptr, nullptr, nullptr, selected);
nullptr, selected);
cb(moe_out, "ffn_moe_out", il);
ggml_tensor * shared = build_ffn(cur,
+1 -6
View File
@@ -116,10 +116,5 @@ ggml_tensor * llm_build_lfm_base::build_lfm_moe_ffn(ggml_tensor * cur,
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
static_cast<llama_expert_gating_func_type>(hparams.expert_gating_func),
il,
nullptr,
nullptr,
model.layers[il].ffn_up_exps_s,
model.layers[il].ffn_gate_exps_s,
model.layers[il].ffn_down_exps_s);
il);
}
+1 -5
View File
@@ -198,11 +198,7 @@ llama_model_llama::graph<embed>::graph(const llm_graph_params & params) : llm_gr
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
il,
nullptr, nullptr,
model.layers[il].ffn_up_exps_s,
model.layers[il].ffn_gate_exps_s,
model.layers[il].ffn_down_exps_s);
il);
cb(cur, "ffn_moe_out", il);
}
cur = ggml_add(ctx0, cur, ffn_inp);
+2 -10
View File
@@ -199,11 +199,7 @@ llama_model_mellum::graph<iswa>::graph(const llm_graph_params & params) : llm_gr
LLM_FFN_SILU, hparams.expert_weights_norm,
hparams.expert_weights_scale,
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
nullptr, nullptr,
model.layers[il].ffn_up_exps_s,
model.layers[il].ffn_gate_exps_s,
model.layers[il].ffn_down_exps_s);
il);
cb(moe_out, "ffn_moe_out", il);
cur = moe_out;
@@ -351,11 +347,7 @@ llama_model_mellum::graph_mtp<iswa>::graph_mtp(const llm_graph_params & params)
LLM_FFN_SILU, hparams.expert_weights_norm,
hparams.expert_weights_scale,
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
nullptr, nullptr,
layer.ffn_up_exps_s,
layer.ffn_gate_exps_s,
layer.ffn_down_exps_s);
il);
cb(cur, "mtp_ffn_moe_out", il);
cur = ggml_add(ctx0, cur, ffn_inp);
+1 -1
View File
@@ -329,7 +329,7 @@ llama_model_metis16::graph::graph(const llm_graph_params & params) :
ggml_tensor * moe_out = build_moe_ffn(latent, nullptr, nullptr, nullptr, layer.ffn_down_exps, nullptr,
n_exp, n_used, LLM_FFN_SILU, false, 0.0f, LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX_WEIGHT, il_p,
logits, layer.ffn_gate_up_exps, nullptr, nullptr, nullptr, selected);
logits, layer.ffn_gate_up_exps, selected);
ggml_tensor * shexp = build_ffn(latent,
layer.ffn_up_shexp, NULL, NULL,
layer.ffn_gate_shexp, NULL, NULL,
+1 -4
View File
@@ -572,10 +572,7 @@ llama_model_minimax_m3::graph::graph(const llm_graph_params & params) : llm_grap
LLM_FFN_SWIGLU_OAI_MOE, hparams.expert_weights_norm,
hparams.expert_weights_scale,
(llama_expert_gating_func_type) hparams.expert_gating_func,
il, nullptr, nullptr,
model.layers[il].ffn_up_exps_s,
model.layers[il].ffn_gate_exps_s,
model.layers[il].ffn_down_exps_s);
il);
cb(moe_out, "ffn_moe_out", il);
// shared expert (swigluoai)
+1 -4
View File
@@ -420,10 +420,7 @@ llama_model_mistral_large4::graph::graph(const llm_graph_params & params) :
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
nullptr,
layer.ffn_gate_up_exps,
layer.ffn_up_exps_s,
layer.ffn_gate_exps_s,
layer.ffn_down_exps_s);
layer.ffn_gate_up_exps);
cb(moe_out, "ffn_moe_out", il);
ggml_tensor * ffn_shexp = build_ffn(cur,
+1 -5
View File
@@ -248,11 +248,7 @@ llama_model_mistral3::graph::graph(const llm_graph_params & params) : llm_graph_
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
il,
nullptr, nullptr,
model.layers[il].ffn_up_exps_s,
model.layers[il].ffn_gate_exps_s,
model.layers[il].ffn_down_exps_s);
il);
cb(cur, "ffn_moe_out", il);
}
cur = ggml_add(ctx0, cur, ffn_inp);
+1 -1
View File
@@ -180,7 +180,7 @@ llama_model_muse_spark::graph::graph(const llm_graph_params & params)
(llama_expert_gating_func_type) hparams.expert_gating_func,
il,
router_logits,
nullptr, nullptr, nullptr, nullptr, nullptr,
nullptr, nullptr,
model.layers[il].ffn_post_exp_norm, hparams.f_post_norm_eps);
cb(moe_out, "ffn_moe_out", il);
+1 -4
View File
@@ -118,10 +118,7 @@ llama_model_nemotron_h_moe::graph_mtp::graph_mtp(const llm_graph_params & params
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,
il,
router_logits, nullptr,
layer.ffn_up_exps_s,
nullptr, // no gate
layer.ffn_down_exps_s);
router_logits);
cb(moe_out, "mtp_ffn_moe_out", il);
if (layer.ffn_latent_up) {
+1 -4
View File
@@ -322,10 +322,7 @@ ggml_tensor * llama_model_nemotron_h::graph::build_ffn_layer(ggml_tensor * cur,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,
il,
router_logits, nullptr,
model.layers[il].ffn_up_exps_s,
nullptr, // no gate
model.layers[il].ffn_down_exps_s);
router_logits);
cb(moe_out, "ffn_moe_out", il);
if (model.layers[il].ffn_latent_up) {
+2 -8
View File
@@ -479,10 +479,7 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_ffn(ggml_tensor * cur, c
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
nullptr, model.layers[il].ffn_gate_up_exps,
model.layers[il].ffn_up_exps_s,
model.layers[il].ffn_gate_exps_s,
model.layers[il].ffn_down_exps_s);
nullptr, model.layers[il].ffn_gate_up_exps);
cb(moe_out, "ffn_moe_out", il);
// Add shared experts if present - following Qwen3Next reference implementation
@@ -651,10 +648,7 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llm_graph_params & params)
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
nullptr, layer.ffn_gate_up_exps,
layer.ffn_up_exps_s,
layer.ffn_gate_exps_s,
layer.ffn_down_exps_s);
nullptr, layer.ffn_gate_up_exps);
cb(moe_out, "mtp_ffn_moe_out", il);
if (layer.ffn_up_shexp != nullptr) {
+1 -5
View File
@@ -136,11 +136,7 @@ llama_model_qwen3moe::graph::graph(const llm_graph_params & params) : llm_graph_
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
il,
nullptr, nullptr,
model.layers[il].ffn_up_exps_s,
model.layers[il].ffn_gate_exps_s,
model.layers[il].ffn_down_exps_s);
il);
cb(moe_out, "ffn_moe_out", il);
cur = moe_out;
+2 -8
View File
@@ -561,10 +561,7 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_ffn(ggml_tensor * cur, c
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
nullptr, model.layers[il].ffn_gate_up_exps,
model.layers[il].ffn_up_exps_s,
model.layers[il].ffn_gate_exps_s,
model.layers[il].ffn_down_exps_s);
nullptr, model.layers[il].ffn_gate_up_exps);
cb(moe_out, "ffn_moe_out", il);
// Add shared experts if present - following Qwen3Next reference implementation
@@ -748,10 +745,7 @@ llama_model_qwen3next::graph_mtp::graph_mtp(const llm_graph_params & params)
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
nullptr, layer.ffn_gate_up_exps,
layer.ffn_up_exps_s,
layer.ffn_gate_exps_s,
layer.ffn_down_exps_s);
nullptr, layer.ffn_gate_up_exps);
cb(moe_out, "mtp_ffn_moe_out", il);
if (layer.ffn_up_shexp != nullptr) {
+1 -4
View File
@@ -1230,10 +1230,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_ffn(ggml_tensor * cur, co
LLM_FFN_SILU, true,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
nullptr, model.layers[il].ffn_gate_up_exps,
model.layers[il].ffn_up_exps_s,
model.layers[il].ffn_gate_exps_s,
model.layers[il].ffn_down_exps_s);
nullptr, model.layers[il].ffn_gate_up_exps);
cb(moe_out, "ffn_moe_out", il);
// shared experts, as in the Qwen3Next reference
-3
View File
@@ -277,9 +277,6 @@ ggml_tensor * llama_model_zaya1::graph::build_router_moe(const llama_model & mod
il,
logits,
layer.ffn_gate_up_exps,
nullptr,
nullptr,
nullptr,
expert);
return ggml_mul(ctx0, moe, keep);
+3 -2
View File
@@ -41,6 +41,7 @@ cohere2 ,0 ,any ,NORM+MUL , 3
cohere2moe ,1 ,any ,ADD+ADD , 2
cohere2moe ,1 ,any ,MUL+ADD , 2
cohere2moe ,1 ,any ,RMS_NORM+MUL , 3
command-r ,0 ,any ,ADD+ADD , 2
command-r ,0 ,any ,NORM+MUL , 3
dbrx ,0 ,any ,MUL+ADD , 4
dbrx ,0 ,any ,NORM+MUL , 5
@@ -207,6 +208,7 @@ llama ,0 ,any ,MUL+ADD , 4
llama ,0 ,any ,RMS_NORM+MUL , 5
llama ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
llama4 ,0 ,any ,ADD+ADD , 2
llama4 ,0 ,any ,MUL+ADD , 4
llama4 ,0 ,any ,RMS_NORM+MUL , 9
maincoder ,0 ,any ,RMS_NORM+MUL , 9
maion-coder ,0 ,any ,MUL+ADD , 4
@@ -257,8 +259,7 @@ moshi ,0 ,any ,RMS_NORM+MUL , 5
mpt ,0 ,any ,NORM+MUL+ADD , 5
muse-glimmer ,0 ,any ,RMS_NORM+MUL , 10
muse-glimmer ,0 ,any ,RMS_NORM+MUL+ADD , 3
muse-spark ,0 ,prefill ,MUL+ADD , 2
muse-spark ,0 ,decode ,MUL+ADD , 4
muse-spark ,0 ,any ,MUL+ADD , 4
muse-spark ,0 ,any ,RMS_NORM+MUL , 11
muse-spark ,0 ,any ,RMS_NORM+MUL+ADD , 3
muse-spark ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
1 # test-fusion baseline for device MTL
41 cohere2moe ,1 ,any ,ADD+ADD , 2
42 cohere2moe ,1 ,any ,MUL+ADD , 2
43 cohere2moe ,1 ,any ,RMS_NORM+MUL , 3
44 command-r ,0 ,any ,ADD+ADD , 2
45 command-r ,0 ,any ,NORM+MUL , 3
46 dbrx ,0 ,any ,MUL+ADD , 4
47 dbrx ,0 ,any ,NORM+MUL , 5
208 llama ,0 ,any ,RMS_NORM+MUL , 5
209 llama ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
210 llama4 ,0 ,any ,ADD+ADD , 2
211 llama4 ,0 ,any ,MUL+ADD , 4
212 llama4 ,0 ,any ,RMS_NORM+MUL , 9
213 maincoder ,0 ,any ,RMS_NORM+MUL , 9
214 maion-coder ,0 ,any ,MUL+ADD , 4
259 mpt ,0 ,any ,NORM+MUL+ADD , 5
260 muse-glimmer ,0 ,any ,RMS_NORM+MUL , 10
261 muse-glimmer ,0 ,any ,RMS_NORM+MUL+ADD , 3
262 muse-spark ,0 ,prefill ,MUL+ADD , 2 muse-spark ,0 ,any ,MUL+ADD , 4
muse-spark ,0 ,decode ,MUL+ADD , 4
263 muse-spark ,0 ,any ,RMS_NORM+MUL , 11
264 muse-spark ,0 ,any ,RMS_NORM+MUL+ADD , 3
265 muse-spark ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2