sycl: fuse UNARY(silu|sigmoid|softplus) + MUL (#26411)
Measured on Arc Pro B70 (Battlemage), Qwen3.6-27B Q4_K_M, -fa on, f16 KV, -b 2048 -ub 2048, llama-bench -r 3, three interleaved A/B rounds: pp2048 1014.70 -> 1018.56 t/s (+0.38%, within run-to-run spread) tg128 23.73 -> 23.86 t/s (+0.57%) tg128 @ d4096 22.71 -> 22.86 t/s (+0.62%)
This commit is contained in:
@@ -1,6 +1,14 @@
|
||||
#include "fusion.hpp"
|
||||
|
||||
bool ggml_sycl_can_fuse(const ggml_cgraph * cgraph, int node_idx, std::initializer_list<enum ggml_op> ops) {
|
||||
#include <algorithm>
|
||||
|
||||
bool ggml_sycl_can_fuse(const ggml_cgraph * cgraph, int node_idx, std::initializer_list<enum ggml_op> ops,
|
||||
std::initializer_list<enum ggml_unary_op> unary_ops) {
|
||||
#ifndef NDEBUG
|
||||
const size_t num_unary = std::count(ops.begin(), ops.end(), GGML_OP_UNARY);
|
||||
GGML_ASSERT(unary_ops.size() == num_unary);
|
||||
#endif
|
||||
|
||||
if (!g_ggml_sycl_enable_fusion) {
|
||||
return false;
|
||||
}
|
||||
@@ -40,5 +48,45 @@ bool ggml_sycl_can_fuse(const ggml_cgraph * cgraph, int node_idx, std::initializ
|
||||
return true;
|
||||
}
|
||||
|
||||
if (ops.size() == 2 && ops.begin()[0] == GGML_OP_UNARY && ops.begin()[1] == GGML_OP_MUL &&
|
||||
unary_ops.size() == 1) {
|
||||
const ggml_tensor * unary = cgraph->nodes[node_idx];
|
||||
const ggml_tensor * mul = cgraph->nodes[node_idx + 1];
|
||||
|
||||
const ggml_unary_op unary_op = ggml_get_unary_op(unary);
|
||||
if (unary_op != unary_ops.begin()[0]) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// the ops ggml_sycl_op_unary_mul_fused() has a kernel for
|
||||
if (unary_op != GGML_UNARY_OP_SILU && unary_op != GGML_UNARY_OP_SIGMOID &&
|
||||
unary_op != GGML_UNARY_OP_SOFTPLUS) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (unary->type != GGML_TYPE_F32 && unary->type != GGML_TYPE_F16) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const ggml_tensor * other = (mul->src[0] == unary) ? mul->src[1] : mul->src[0];
|
||||
if (other->type != unary->type) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// one row stride per source comes from nb[1], so rows must be contiguous and equally
|
||||
// shaped; the destination is written flat, so it must be fully contiguous
|
||||
if (!ggml_is_contiguous_1(unary->src[0]) || !ggml_is_contiguous_1(other) ||
|
||||
!ggml_are_same_shape(other, unary) || !ggml_is_contiguous(mul)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// the 32-bit fastdiv is inexact past 2^31; decline, the unfused path handles it
|
||||
if (ggml_nelements(mul) >= ((int64_t) 1 << 31)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user