diff --git a/ggml/src/ggml-cpu/vec.cpp b/ggml/src/ggml-cpu/vec.cpp index 3918a6f4d..811184d82 100644 --- a/ggml/src/ggml-cpu/vec.cpp +++ b/ggml/src/ggml-cpu/vec.cpp @@ -401,6 +401,10 @@ void ggml_vec_silu_f32(const int n, float * y, const float * x) { for (; i + 3 < n; i += 4) { vst1q_f32(y + i, ggml_v_silu(vld1q_f32(x + i))); } +#elif defined(__VXE__) || defined(__VXE2__) + for (; i + 3 < n; i += 4) { + vec_xst(ggml_v_silu(vec_xl(0, x + i)), 0, y + i); + } #elif defined(__riscv_v_intrinsic) for (int vl; i < n; i += vl) { vl = __riscv_vsetvl_e32m2(n - i); @@ -438,6 +442,10 @@ void ggml_vec_swiglu_f32(const int n, float * y, const float * x, const float * for (; i + 3 < n; i += 4) { vst1q_f32(y + i, vmulq_f32(ggml_v_silu(vld1q_f32(x + i)), vld1q_f32(g + i))); } +#elif defined(__VXE__) || defined(__VXE2__) + for (; i + 3 < n; i += 4) { + vec_xst(vec_mul(ggml_v_silu(vec_xl(0, x + i)), vec_xl(0, g + i)), 0, y + i); + } #elif defined(__riscv_v_intrinsic) for (int vl; i < n; i += vl) { vl = __riscv_vsetvl_e32m2(n - i); @@ -581,6 +589,12 @@ ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float vst1q_f32(y + i, val); sum += (ggml_float)vaddvq_f32(val); } +#elif defined(__VXE__) || defined(__VXE2__) + for (; i + 3 < n; i += 4) { + float32x4_t v_val = ggml_v_expf(vec_sub(vec_xl(0, x + i), vec_splats(max))); + vec_xst(v_val, 0, y + i); + sum += (ggml_float)vec_hsum_f32x4(v_val); + } #elif defined(__riscv_v_intrinsic) vfloat64m1_t vsum = __riscv_vfmv_v_f_f64m1(0, 1); for (int avl; i < n; i += avl) { diff --git a/ggml/src/ggml-cpu/vec.h b/ggml/src/ggml-cpu/vec.h index 59fa88f2c..db9a38c09 100644 --- a/ggml/src/ggml-cpu/vec.h +++ b/ggml/src/ggml-cpu/vec.h @@ -1334,6 +1334,46 @@ inline static __m128 ggml_v_silu(__m128 x) { return _mm_div_ps(x, one_plus_exp_neg_x); } +#elif defined(__VXE__) || defined(__VXE2__) + +// adapted from arm limited optimized routine +// the maximum error is 1.45358 plus 0.5 ulps +// numbers above 88.38 will flush to infinity +// numbers beneath -103.97 will flush to zero +inline static float32x4_t ggml_v_expf(float32x4_t v_x) { + const float32x4_t v_r = vec_splats(0x1.8p23f); + const float32x4_t v_z = vec_madd(v_x, vec_splats(0x1.715476p+0f), v_r); + const float32x4_t v_n = vec_sub(v_z, v_r); + const float32x4_t v_b = vec_nmsub(v_n, vec_splats(0x1.7f7d1cp-20f), + vec_nmsub(v_n, vec_splats(0x1.62e4p-1f), v_x)); + const uint32x4_t v_e = vec_sl((uint32x4_t)v_z, 23); + const float32x4_t v_k = (float32x4_t)vec_add(v_e, (uint32x4_t)vec_splats(1.0f)); + const uint32x4_t v_c = (uint32x4_t)vec_cmpgt(vec_abs(v_n), vec_splats(126.0f)); + const float32x4_t v_u = vec_mul(v_b, v_b); + const float32x4_t v_j = vec_madd( + vec_madd(vec_madd(vec_splats(0x1.0e4020p-7f), v_b, vec_splats(0x1.573e2ep-5f)), v_u, + vec_madd(vec_splats(0x1.555e66p-3f), v_b, vec_splats(0x1.fffdb6p-2f))), + v_u, vec_mul(vec_splats(0x1.ffffecp-1f), v_b)); + if (vec_all_eq(v_c, vec_splats(UINT32_C(0)))) { + return vec_madd(v_j, v_k, v_k); + } + const uint32x4_t v_d = vec_and((uint32x4_t)vec_cmple(v_n, vec_splats(0.0f)), vec_splats(UINT32_C(0x82000000))); + const float32x4_t v_s1 = (float32x4_t)vec_add(v_d, vec_splats(UINT32_C(0x7f000000))); + const float32x4_t v_s2 = (float32x4_t)vec_sub(v_e, v_d); + return vec_sel(vec_sel(vec_madd(v_k, v_j, v_k), vec_mul(vec_madd(v_s2, v_j, v_s2), v_s1), v_c), + vec_mul(v_s1, v_s1), (uint32x4_t)vec_cmpgt(vec_abs(v_n), vec_splats(192.0f))); +} + +// computes silu x/(1+exp(-x)) in single precision vector +inline static float32x4_t ggml_v_silu(float32x4_t v_x) { + const float32x4_t v_one = vec_splats(1.0f); + const float32x4_t v_zero = vec_splats(0.0f); + const float32x4_t v_neg_x = vec_sub(v_zero, v_x); + const float32x4_t v_exp_neg_x = ggml_v_expf(v_neg_x); + const float32x4_t v_one_plus_exp_neg_x = vec_add(v_one, v_exp_neg_x); + return vec_div(v_x, v_one_plus_exp_neg_x); +} + #elif defined(__riscv_v_intrinsic) // adapted from arm limited optimized routine @@ -1386,7 +1426,7 @@ inline static vfloat32m2_t ggml_v_silu_m2(vfloat32m2_t x, int vl) { return __riscv_vfdiv_vv_f32m2(x, one_plus_exp_neg_x, vl); } -#endif // __ARM_NEON / __AVX2__ / __SSE2__ / __riscv_v_intrinsic +#endif // __ARM_NEON / __AVX2__ / __SSE2__ / __VXE__ / __riscv_v_intrinsic inline static void ggml_vec_silu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) { for (int i = 0; i < n; ++i) {