metal : fuse DeepSeek-V4/V4.1 and GLM MoE routing (#31615)
* metal : support scale-bias routing norm in the TOPK_MOE fusion The DeepSeek-V4.1 graph normalizes the routing weights with a scale+bias (sum + 1e-20) instead of the lower clamp used by the other models. Add norm_bias/norm_scale_bias fusion chains that accept a SCALE node with a unit scale factor at the clamp position, pass an additive bias in the kernel args, and apply it as max(wt_sum, clamp) + bias so both variants share one kernel. Cover the new chains in test-backend-ops. Assisted-by: dsh:llama.cpp/ggml-org/GLM-5.3-Flash-GGUF (Mac Studio, M5 Ultra, 256GB) * metal : fuse sqrt(softplus) MoE routing with a selection bias Add TOPK_MOE chains anchored at the softplus node (UNARY+SQRT+ADD+ ARGSORT+GET_ROWS plus optional norm/scale) so the DeepSeek-V4/V4.1 routing fuses on Metal. The kernel scores with sqrt(softplus) and ranks selection-biased keys while the gathered weights and the norm stay unbiased, matching the reference graph. Dispatch hooks at the softplus node, cover the norm-bias combination in test-backend-ops and record the new deepseek4/deepseek41 fusion counters. Assisted-by: dsh:llama.cpp/ggml-org/GLM-5.3-Flash-GGUF (Mac Studio, M5 Ultra, 256GB) * metal : fuse sigmoid MoE routing with a selection bias The sigmoid-gated MoE archs (GLM, DeepSeek-V2/V3.2, ...) rank sigmoid(logits) + exp_probs_b with an optional weights-norm, but only the softmax and sqrt(softplus) routing chains were fusable. - add 6 sigmoid-anchored TOPK_MOE chains: sigmoid + ADD + ARGSORT + GET_ROWS with optional SCALE and SUM_ROWS + CLAMP/SCALE + DIV norm tail; the selection-bias ADD is required, same as the sqrt(softplus) chains - generalize the gating function constant from a sqrt(softplus) bool to an int (0 = softmax, 1 = sqrt(softplus), 2 = sigmoid); sigmoid scores are 1 / (1 + exp(-logits)), matching the CPU, Metal unary and CUDA kernels - anchor the unary dispatch hook at sigmoid nodes too - test-backend-ops: 4 sigmoid routing cases with and without a weights scale - re-record tests/fusion/MTL.csv: the sigmoid archs (glm5-next, deepseek2/32, kimi, minimax, ...) now fuse their routing, and muse-spark prefill drops one MOE_REDUCE fire because the reordered graph allocates the routing weights inside the reduce output range, which the dst/input overlap check correctly declines Assisted-by: dsh:llama.cpp/ggml-org/GLM-5.3-Flash-GGUF (Mac Studio, M5 Ultra, 256GB) * cont : remove low-signal comments * ocd : valign ggml_metal_fusions
This commit is contained in:
1 parent
15e50990bb
commit
ea40a3a1e1
8 files changed
+348
-106
No files matched your search
@@ -7551,6 +7551,7 @@ struct test_topk_moe : public test_case {
|
||||
const int n_expert_groups;
|
||||
ggml_tensor * logits {};
|
||||
ggml_tensor * exp_probs_b {};
|
||||
const bool norm_bias;
|
||||
ggml_tensor * weights {};
|
||||
ggml_tensor * selected_experts {};
|
||||
|
||||
@@ -7560,20 +7561,23 @@ struct test_topk_moe : public test_case {
|
||||
bool bias_probs = false,
|
||||
MoeGatingFunc gating_func = GATING_FUNC_SOFTMAX,
|
||||
float scale_w = 0.0f,
|
||||
int n_expert_groups = 1) :
|
||||
int n_expert_groups = 1,
|
||||
bool norm_bias = false) :
|
||||
ne(ne),
|
||||
n_expert_used(n_expert_used),
|
||||
with_norm(with_norm),
|
||||
bias_probs(bias_probs),
|
||||
gating_func(gating_func),
|
||||
scale_w(scale_w),
|
||||
n_expert_groups(n_expert_groups) {
|
||||
n_expert_groups(n_expert_groups),
|
||||
norm_bias(norm_bias) {
|
||||
GGML_ASSERT(n_expert_used <= ne[0]);
|
||||
GGML_ASSERT(ne[0] % n_expert_groups == 0);
|
||||
GGML_ASSERT(n_expert_used <= ne[0] / n_expert_groups);
|
||||
GGML_ASSERT(!norm_bias || with_norm);
|
||||
}
|
||||
|
||||
std::string vars() override { return VARS_TO_STR7(ne, n_expert_used, with_norm, bias_probs, gating_func, scale_w, n_expert_groups); }
|
||||
std::string vars() override { return VARS_TO_STR8(ne, n_expert_used, with_norm, norm_bias, bias_probs, gating_func, scale_w, n_expert_groups); }
|
||||
|
||||
std::string op_desc(ggml_tensor * t) override {
|
||||
GGML_UNUSED(t);
|
||||
@@ -7731,7 +7735,11 @@ struct test_topk_moe : public test_case {
|
||||
ggml_tensor * weights_sum = ggml_sum_rows(ctx, weights); // [1, n_tokens]
|
||||
ggml_set_name(weights_sum, "weights_sum");
|
||||
|
||||
weights_sum = ggml_clamp(ctx, weights_sum, 6.103515625e-5, INFINITY);
|
||||
if (norm_bias) {
|
||||
weights_sum = ggml_scale_bias(ctx, weights_sum, 1.0f, 1e-20f);
|
||||
} else {
|
||||
weights_sum = ggml_clamp(ctx, weights_sum, 6.103515625e-5, INFINITY);
|
||||
}
|
||||
weights = ggml_div(ctx, weights, weights_sum); // [n_expert_used, n_tokens]
|
||||
weights = ggml_reshape_3d(ctx, weights, 1, n_expert_used, n_tokens);
|
||||
}
|
||||
@@ -12871,6 +12879,22 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
}
|
||||
}
|
||||
|
||||
// the sum is adjusted with an additive bias instead of a lower clamp (DeepSeek-V4.1)
|
||||
for (bool bias_probs : {false, true}) {
|
||||
for (float scale_w : {0.0f, 2.0f}) {
|
||||
test_cases.emplace_back(new test_topk_moe({8, 22, 1, 1}, 4, true, bias_probs, GATING_FUNC_SOFTMAX, scale_w, 1, true));
|
||||
test_cases.emplace_back(new test_topk_moe({384, 1, 1, 1}, 6, true, bias_probs, GATING_FUNC_SOFTMAX, scale_w, 1, true));
|
||||
test_cases.emplace_back(new test_topk_moe({8, 22, 1, 1}, 4, true, bias_probs, GATING_FUNC_SQRT_SOFTPLUS, scale_w, 1, true));
|
||||
test_cases.emplace_back(new test_topk_moe({384, 1, 1, 1}, 6, true, bias_probs, GATING_FUNC_SQRT_SOFTPLUS, scale_w, 1, true));
|
||||
}
|
||||
}
|
||||
|
||||
// sigmoid scoring always ranks a selection bias (GLM, DeepSeek-V2/V3.2)
|
||||
for (float scale_w : {0.0f, 2.0f}) {
|
||||
test_cases.emplace_back(new test_topk_moe({8, 22, 1, 1}, 4, true, true, GATING_FUNC_SIGMOID, scale_w, 1, true));
|
||||
test_cases.emplace_back(new test_topk_moe({384, 1, 1, 1}, 6, true, true, GATING_FUNC_SIGMOID, scale_w, 1, true));
|
||||
}
|
||||
|
||||
// Cover the supported boundaries, common k = 8 shapes, interleaved views and adds, and k = 16 fallback.
|
||||
test_cases.emplace_back(new test_moe_reduce(63, 2, 17));
|
||||
test_cases.emplace_back(new test_moe_reduce(2048, 8, 128));
|
||||
|
||||
Reference in new issue
Block a user