vulkan: extend topk_moe fusion to support sqrt(softplus) (#26124)

This commit is contained in:
Jeff Bolz
2026-08-01 14:18:07 -05:00
committed by GitHub
parent 815a2a5915
commit a7a6d0d269
2 changed files with 84 additions and 11 deletions
@@ -10,6 +10,7 @@
#define GATING_FUNC_SOFTMAX 0
#define GATING_FUNC_SIGMOID 1
#define GATING_FUNC_SOFTMAX_WEIGHT 2
#define GATING_FUNC_SQRT_SOFTPLUS 3
layout (push_constant) uniform parameter
{
@@ -120,6 +121,13 @@ void main() {
const uint expert = i + lane;
probs[i / WARP_SIZE] = (n_experts % WARP_SIZE == 0 || expert < n_experts) ? 1.f / (1.f + exp(-probs[i / WARP_SIZE])) : -INFINITY;
}
} else if (gating_func == GATING_FUNC_SQRT_SOFTPLUS) {
[[unroll]]
for (uint i = 0; i < n_experts; i += WARP_SIZE) {
const uint expert = i + lane;
const float val = probs[i / WARP_SIZE];
probs[i / WARP_SIZE] = (n_experts % WARP_SIZE == 0 || expert < n_experts) ? sqrt(val > 20.0f ? val : log(1.0f + exp(val))) : -INFINITY;
}
}
float selection_probs[experts_per_thread];