ggml-cpu: vectorize ggml_v_silu and ggml_v_expf (#30791)

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>
This commit is contained in:
Aaron Teo authored and GitHub committed 2026-10-19 09:59:49 +02:00
1 parent 4c1004c492
commit 42b70032b1
2 files changed
+55 -1

No files matched your search

+14
View File
@@ -401,6 +401,10 @@ void ggml_vec_silu_f32(const int n, float * y, const float * x) {
for (; i + 3 < n; i += 4) {
vst1q_f32(y + i, ggml_v_silu(vld1q_f32(x + i)));
}
#elif defined(__VXE__) || defined(__VXE2__)
for (; i + 3 < n; i += 4) {
vec_xst(ggml_v_silu(vec_xl(0, x + i)), 0, y + i);
}
#elif defined(__riscv_v_intrinsic)
for (int vl; i < n; i += vl) {
vl = __riscv_vsetvl_e32m2(n - i);
@@ -438,6 +442,10 @@ void ggml_vec_swiglu_f32(const int n, float * y, const float * x, const float *
for (; i + 3 < n; i += 4) {
vst1q_f32(y + i, vmulq_f32(ggml_v_silu(vld1q_f32(x + i)), vld1q_f32(g + i)));
}
#elif defined(__VXE__) || defined(__VXE2__)
for (; i + 3 < n; i += 4) {
vec_xst(vec_mul(ggml_v_silu(vec_xl(0, x + i)), vec_xl(0, g + i)), 0, y + i);
}
#elif defined(__riscv_v_intrinsic)
for (int vl; i < n; i += vl) {
vl = __riscv_vsetvl_e32m2(n - i);
@@ -581,6 +589,12 @@ ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float
vst1q_f32(y + i, val);
sum += (ggml_float)vaddvq_f32(val);
}
#elif defined(__VXE__) || defined(__VXE2__)
for (; i + 3 < n; i += 4) {
float32x4_t v_val = ggml_v_expf(vec_sub(vec_xl(0, x + i), vec_splats(max)));
vec_xst(v_val, 0, y + i);
sum += (ggml_float)vec_hsum_f32x4(v_val);
}
#elif defined(__riscv_v_intrinsic)
vfloat64m1_t vsum = __riscv_vfmv_v_f_f64m1(0, 1);
for (int avl; i < n; i += avl) {
+41 -1
View File
@@ -1334,6 +1334,46 @@ inline static __m128 ggml_v_silu(__m128 x) {
return _mm_div_ps(x, one_plus_exp_neg_x);
}
#elif defined(__VXE__) || defined(__VXE2__)
// adapted from arm limited optimized routine
// the maximum error is 1.45358 plus 0.5 ulps
// numbers above 88.38 will flush to infinity
// numbers beneath -103.97 will flush to zero
inline static float32x4_t ggml_v_expf(float32x4_t v_x) {
const float32x4_t v_r = vec_splats(0x1.8p23f);
const float32x4_t v_z = vec_madd(v_x, vec_splats(0x1.715476p+0f), v_r);
const float32x4_t v_n = vec_sub(v_z, v_r);
const float32x4_t v_b = vec_nmsub(v_n, vec_splats(0x1.7f7d1cp-20f),
vec_nmsub(v_n, vec_splats(0x1.62e4p-1f), v_x));
const uint32x4_t v_e = vec_sl((uint32x4_t)v_z, 23);
const float32x4_t v_k = (float32x4_t)vec_add(v_e, (uint32x4_t)vec_splats(1.0f));
const uint32x4_t v_c = (uint32x4_t)vec_cmpgt(vec_abs(v_n), vec_splats(126.0f));
const float32x4_t v_u = vec_mul(v_b, v_b);
const float32x4_t v_j = vec_madd(
vec_madd(vec_madd(vec_splats(0x1.0e4020p-7f), v_b, vec_splats(0x1.573e2ep-5f)), v_u,
vec_madd(vec_splats(0x1.555e66p-3f), v_b, vec_splats(0x1.fffdb6p-2f))),
v_u, vec_mul(vec_splats(0x1.ffffecp-1f), v_b));
if (vec_all_eq(v_c, vec_splats(UINT32_C(0)))) {
return vec_madd(v_j, v_k, v_k);
}
const uint32x4_t v_d = vec_and((uint32x4_t)vec_cmple(v_n, vec_splats(0.0f)), vec_splats(UINT32_C(0x82000000)));
const float32x4_t v_s1 = (float32x4_t)vec_add(v_d, vec_splats(UINT32_C(0x7f000000)));
const float32x4_t v_s2 = (float32x4_t)vec_sub(v_e, v_d);
return vec_sel(vec_sel(vec_madd(v_k, v_j, v_k), vec_mul(vec_madd(v_s2, v_j, v_s2), v_s1), v_c),
vec_mul(v_s1, v_s1), (uint32x4_t)vec_cmpgt(vec_abs(v_n), vec_splats(192.0f)));
}
// computes silu x/(1+exp(-x)) in single precision vector
inline static float32x4_t ggml_v_silu(float32x4_t v_x) {
const float32x4_t v_one = vec_splats(1.0f);
const float32x4_t v_zero = vec_splats(0.0f);
const float32x4_t v_neg_x = vec_sub(v_zero, v_x);
const float32x4_t v_exp_neg_x = ggml_v_expf(v_neg_x);
const float32x4_t v_one_plus_exp_neg_x = vec_add(v_one, v_exp_neg_x);
return vec_div(v_x, v_one_plus_exp_neg_x);
}
#elif defined(__riscv_v_intrinsic)
// adapted from arm limited optimized routine
@@ -1386,7 +1426,7 @@ inline static vfloat32m2_t ggml_v_silu_m2(vfloat32m2_t x, int vl) {
return __riscv_vfdiv_vv_f32m2(x, one_plus_exp_neg_x, vl);
}
#endif // __ARM_NEON / __AVX2__ / __SSE2__ / __riscv_v_intrinsic
#endif // __ARM_NEON / __AVX2__ / __SSE2__ / __VXE__ / __riscv_v_intrinsic
inline static void ggml_vec_silu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {
for (int i = 0; i < n; ++i) {