vulkan: extend topk_moe fusion to support sqrt(softplus) (#26124)
This commit is contained in:
@@ -10,6 +10,7 @@
|
||||
#define GATING_FUNC_SOFTMAX 0
|
||||
#define GATING_FUNC_SIGMOID 1
|
||||
#define GATING_FUNC_SOFTMAX_WEIGHT 2
|
||||
#define GATING_FUNC_SQRT_SOFTPLUS 3
|
||||
|
||||
layout (push_constant) uniform parameter
|
||||
{
|
||||
@@ -120,6 +121,13 @@ void main() {
|
||||
const uint expert = i + lane;
|
||||
probs[i / WARP_SIZE] = (n_experts % WARP_SIZE == 0 || expert < n_experts) ? 1.f / (1.f + exp(-probs[i / WARP_SIZE])) : -INFINITY;
|
||||
}
|
||||
} else if (gating_func == GATING_FUNC_SQRT_SOFTPLUS) {
|
||||
[[unroll]]
|
||||
for (uint i = 0; i < n_experts; i += WARP_SIZE) {
|
||||
const uint expert = i + lane;
|
||||
const float val = probs[i / WARP_SIZE];
|
||||
probs[i / WARP_SIZE] = (n_experts % WARP_SIZE == 0 || expert < n_experts) ? sqrt(val > 20.0f ? val : log(1.0f + exp(val))) : -INFINITY;
|
||||
}
|
||||
}
|
||||
|
||||
float selection_probs[experts_per_thread];
|
||||
|
||||
Reference in New Issue
Block a user