Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

sycl: fuse mul_mat(gate) + mul_mat(up) + GLU for q4_K dense FFN (#26779)

Browse Source 
Measured on Arc Pro B70 (Battlemage, Level Zero), llama-bench -r 20, two
interleaved rounds, tg128:

    qwen2.5-3B-Instruct Q4_K_M    154.18 -> 158.53 t/s   +2.8%
    gemma-2-2b-it Q4_K_M          162.45 -> 165.62 t/s   +2.0%

llama-batched-bench on qwen2.5-3B, S_TG by batch size:

      B=1   142.72 -> 147.57 t/s    +3.4%
      B=2   243.72 -> 268.26 t/s   +10.1%
      B=4   359.58 -> 398.02 t/s   +10.7%
      B=8   449.75 -> 505.63 t/s   +12.4%
This commit is contained in:
Titaniumtown
2026-08-14 02:26:23 -04:00
committed by GitHub
parent c6f6a92c55
commit 6509138622
8 changed files with 316 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-sycl/element_wise.cpp
-37
View File
@@ -81,43 +81,6 @@ static __dpct_inline__ T op_elu(T x) {
return (x > static_cast<T>(0.f)) ? x : op_expm1(x);
}
template<typename T>
static __dpct_inline__ T op_tanh(T x) {
if constexpr (std::is_same_v<T, sycl::ext::oneapi::bfloat16>) {
constexpr int ver = __INTEL_LLVM_COMPILER;
#if defined(__INTEL_LLVM_COMPILER) && (__INTEL_LLVM_COMPILER >= 20260000)
return sycl::ext::oneapi::experimental::tanh(x);
#else
return static_cast<T>(sycl::tanh(static_cast<float>(x)));
#endif
} else {
return sycl::tanh(x);
}
}
template<typename T>
static __dpct_inline__ T op_gelu(T x) {
const T GELU_COEF_A = static_cast<T>(0.044715f);
const T SQRT_2_OVER_PI = static_cast<T>(0.79788456080286535587989211986876f);
return static_cast<T>(0.5f) * x *
(static_cast<T>(1.0f) +
op_tanh(SQRT_2_OVER_PI * x * (static_cast<T>(1.0f) + GELU_COEF_A * x * x)));
}
template<typename T>
static __dpct_inline__ T op_exp(T x) {
if constexpr (std::is_same_v<T, sycl::ext::oneapi::bfloat16>) {
return sycl::ext::oneapi::experimental::exp(x);
} else {
return sycl::exp(x);
}
}
template<typename T>
static __dpct_inline__ T op_silu(T x) {
return x / (static_cast<T>(1.0f) + op_exp(-x));
}
template<typename T>
static __dpct_inline__ T op_erf(T x) {
if constexpr (std::is_same_v<T, sycl::ext::oneapi::bfloat16>) {