llamafile: impl bf16 and q8_0 vxe (#31866)

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>
This commit is contained in:
Aaron Teo authored and GitHub committed 2026-11-08 16:44:25 +02:00
1 parent 57d04820df
commit a3c7cf08f9
1 file changed
+107
+107
View File
@@ -326,6 +326,12 @@ template <> inline float32x4_t load(const ggml_fp16_t * p) {
return vec_xl(0, (const float *)(tmp));
}
template <> inline float32x4_t load(const ggml_bf16_t * p) {
uint64_t bits;
memcpy(&bits, p, sizeof(bits));
const uint64x2_t v_bits = { bits, bits };
return GGML_BF16_TO_F32_HI((GGML_BF16_VEC) v_bits);
}
template <> inline float32x4_t load(const float * p) {
return vec_xl(0, p);
}
@@ -1302,6 +1308,88 @@ class tinyBLAS_RVV {
//////////////////////////////////////////////////////////////////////////////////////////
// QUANT ZERO MATRIX MULTIPLICATION
#if defined(__VXE__) || defined(__VXE2__)
template <typename TA>
class tinyBLAS_Q0_VXE {
static_assert(sizeof(TA::qs) == QK8_0, "VXE weight loads require 32 quantized bytes per block");
public:
tinyBLAS_Q0_VXE(int64_t k,
const TA * A, int64_t lda,
const block_q8_0 * B, int64_t ldb,
float * C, int64_t ldc,
int32_t ith, int32_t nth)
: A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
}
void matmul(int64_t m, int64_t n) {
constexpr int32_t tile_size = 2;
const int64_t row_tiles = m / tile_size;
const int64_t tiles = row_tiles * (n / tile_size);
const int64_t duty = (tiles + nth - 1) / nth;
const int64_t start = duty * ith;
const int64_t end = MIN(start + duty, tiles);
for (int64_t job = start; job < end; ++job) {
const int64_t ii = tile_size * (job % row_tiles);
const int64_t jj = tile_size * (job / row_tiles);
float32x4_t v_sum[tile_size][tile_size] = {};
for (int64_t l = 0; l < k; ++l) {
int8x16_t v_xl[tile_size], v_xh[tile_size], v_yl[tile_size], v_yh[tile_size];
float x_scale[tile_size], y_scale[tile_size];
for (int32_t i = 0; i < tile_size; ++i) {
const TA & x = A[lda * (ii + i) + l];
const block_q8_0 & y = B[ldb * (jj + i) + l];
const int8_t * x_qs = x.qs;
const int8_t * y_qs = y.qs;
v_xl[i] = vec_xl(0, x_qs);
v_xh[i] = vec_xl(0, x_qs + QK8_0 / 2);
v_yl[i] = vec_xl(0, y_qs);
v_yh[i] = vec_xl(0, y_qs + QK8_0 / 2);
x_scale[i] = unhalf(x.d);
y_scale[i] = unhalf(y.d);
}
for (int32_t j = 0; j < tile_size; ++j) {
for (int32_t i = 0; i < tile_size; ++i) {
const int32x4_t v_dot = dot_s8_wide(v_xl[i], v_yl[j]) + dot_s8_wide(v_xh[i], v_yh[j]);
const float32x4_t v_scale = vec_splats(x_scale[i] * y_scale[j]);
v_sum[j][i] = vec_madd(vec_float(v_dot), v_scale, v_sum[j][i]);
}
}
}
for (int32_t j = 0; j < tile_size; ++j) {
for (int32_t i = 0; i < tile_size; ++i) {
C[ldc * (jj + j) + ii + i] = hsum(v_sum[j][i]);
}
}
}
}
private:
static int32x4_t dot_s8_wide(int8x16_t v_x, int8x16_t v_y) {
const int16x8_t v_even = vec_mule(v_x, v_y);
const int16x8_t v_odd = vec_mulo(v_x, v_y);
// Two (-128) * (-128) products can overflow a signed 16-bit sum.
return vec_unpackh(v_even) + vec_unpackl(v_even) + vec_unpackh(v_odd) + vec_unpackl(v_odd);
}
const TA * const A;
const block_q8_0 * const B;
float * const C;
const int64_t k;
const int64_t lda;
const int64_t ldb;
const int64_t ldc;
const int32_t ith;
const int32_t nth;
};
#endif
#if defined(__ARM_FEATURE_DOTPROD)
template <typename TA>
class tinyBLAS_Q0_ARM {
@@ -4006,6 +4094,14 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64
(float *)C, ldc};
return tb.matmul(m, n);
}
#elif defined(__VXE__) || defined(__VXE2__)
if (Btype == GGML_TYPE_BF16) {
tinyBLAS<4, float32x4_t, float32x4_t, ggml_bf16_t, ggml_bf16_t, float> tb{ params, k,
(const ggml_bf16_t *)A, lda,
(const ggml_bf16_t *)B, ldb,
(float *)C, ldc};
return tb.matmul(m, n);
}
#elif defined(__MMA__)
if (k % 8) {
return false;
@@ -4162,6 +4258,17 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64
params->ith, params->nth};
tb.matmul(m, n);
return true;
#elif defined(__VXE__) || defined(__VXE2__)
if (m % 2 || n % 2) {
return false;
}
tinyBLAS_Q0_VXE<block_q8_0> tb{
k, (const block_q8_0 *)A, lda,
(const block_q8_0 *)B, ldb,
(float *)C, ldc,
params->ith, params->nth};
tb.matmul(m, n);
return true;
#elif defined(__MMA__)
//TO-DO: Remove this condition once gemv forwarding is enabled.
if (n < 8 && n != 4)