From a3c7cf08f98fd769f246fcb72be8872d54ff3c35 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Sun, 8 Nov 2026 22:44:25 +0800 Subject: [PATCH] llamafile: impl bf16 and q8_0 vxe (#31866) Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/llamafile/sgemm.cpp | 107 ++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp index 6d39cd6e7..384f7dbe2 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp +++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp @@ -326,6 +326,12 @@ template <> inline float32x4_t load(const ggml_fp16_t * p) { return vec_xl(0, (const float *)(tmp)); } +template <> inline float32x4_t load(const ggml_bf16_t * p) { + uint64_t bits; + memcpy(&bits, p, sizeof(bits)); + const uint64x2_t v_bits = { bits, bits }; + return GGML_BF16_TO_F32_HI((GGML_BF16_VEC) v_bits); +} template <> inline float32x4_t load(const float * p) { return vec_xl(0, p); } @@ -1302,6 +1308,88 @@ class tinyBLAS_RVV { ////////////////////////////////////////////////////////////////////////////////////////// // QUANT ZERO MATRIX MULTIPLICATION +#if defined(__VXE__) || defined(__VXE2__) +template +class tinyBLAS_Q0_VXE { + static_assert(sizeof(TA::qs) == QK8_0, "VXE weight loads require 32 quantized bytes per block"); + + public: + tinyBLAS_Q0_VXE(int64_t k, + const TA * A, int64_t lda, + const block_q8_0 * B, int64_t ldb, + float * C, int64_t ldc, + int32_t ith, int32_t nth) + : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) { + } + + void matmul(int64_t m, int64_t n) { + constexpr int32_t tile_size = 2; + const int64_t row_tiles = m / tile_size; + const int64_t tiles = row_tiles * (n / tile_size); + const int64_t duty = (tiles + nth - 1) / nth; + const int64_t start = duty * ith; + const int64_t end = MIN(start + duty, tiles); + + for (int64_t job = start; job < end; ++job) { + const int64_t ii = tile_size * (job % row_tiles); + const int64_t jj = tile_size * (job / row_tiles); + float32x4_t v_sum[tile_size][tile_size] = {}; + + for (int64_t l = 0; l < k; ++l) { + int8x16_t v_xl[tile_size], v_xh[tile_size], v_yl[tile_size], v_yh[tile_size]; + float x_scale[tile_size], y_scale[tile_size]; + + for (int32_t i = 0; i < tile_size; ++i) { + const TA & x = A[lda * (ii + i) + l]; + const block_q8_0 & y = B[ldb * (jj + i) + l]; + + const int8_t * x_qs = x.qs; + const int8_t * y_qs = y.qs; + v_xl[i] = vec_xl(0, x_qs); + v_xh[i] = vec_xl(0, x_qs + QK8_0 / 2); + v_yl[i] = vec_xl(0, y_qs); + v_yh[i] = vec_xl(0, y_qs + QK8_0 / 2); + x_scale[i] = unhalf(x.d); + y_scale[i] = unhalf(y.d); + } + + for (int32_t j = 0; j < tile_size; ++j) { + for (int32_t i = 0; i < tile_size; ++i) { + const int32x4_t v_dot = dot_s8_wide(v_xl[i], v_yl[j]) + dot_s8_wide(v_xh[i], v_yh[j]); + const float32x4_t v_scale = vec_splats(x_scale[i] * y_scale[j]); + v_sum[j][i] = vec_madd(vec_float(v_dot), v_scale, v_sum[j][i]); + } + } + } + + for (int32_t j = 0; j < tile_size; ++j) { + for (int32_t i = 0; i < tile_size; ++i) { + C[ldc * (jj + j) + ii + i] = hsum(v_sum[j][i]); + } + } + } + } + + private: + static int32x4_t dot_s8_wide(int8x16_t v_x, int8x16_t v_y) { + const int16x8_t v_even = vec_mule(v_x, v_y); + const int16x8_t v_odd = vec_mulo(v_x, v_y); + // Two (-128) * (-128) products can overflow a signed 16-bit sum. + return vec_unpackh(v_even) + vec_unpackl(v_even) + vec_unpackh(v_odd) + vec_unpackl(v_odd); + } + + const TA * const A; + const block_q8_0 * const B; + float * const C; + const int64_t k; + const int64_t lda; + const int64_t ldb; + const int64_t ldc; + const int32_t ith; + const int32_t nth; +}; +#endif + #if defined(__ARM_FEATURE_DOTPROD) template class tinyBLAS_Q0_ARM { @@ -4006,6 +4094,14 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 (float *)C, ldc}; return tb.matmul(m, n); } +#elif defined(__VXE__) || defined(__VXE2__) + if (Btype == GGML_TYPE_BF16) { + tinyBLAS<4, float32x4_t, float32x4_t, ggml_bf16_t, ggml_bf16_t, float> tb{ params, k, + (const ggml_bf16_t *)A, lda, + (const ggml_bf16_t *)B, ldb, + (float *)C, ldc}; + return tb.matmul(m, n); + } #elif defined(__MMA__) if (k % 8) { return false; @@ -4162,6 +4258,17 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 params->ith, params->nth}; tb.matmul(m, n); return true; +#elif defined(__VXE__) || defined(__VXE2__) + if (m % 2 || n % 2) { + return false; + } + tinyBLAS_Q0_VXE tb{ + k, (const block_q8_0 *)A, lda, + (const block_q8_0 *)B, ldb, + (float *)C, ldc, + params->ith, params->nth}; + tb.matmul(m, n); + return true; #elif defined(__MMA__) //TO-DO: Remove this condition once gemv forwarding is enabled. if (n < 8 && n != 4)