llamafile: impl bf16 and q8_0 vxe (#31866)
Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>
This commit is contained in:
1 parent
57d04820df
commit
a3c7cf08f9
1 file changed
+107
@@ -326,6 +326,12 @@ template <> inline float32x4_t load(const ggml_fp16_t * p) {
|
||||
|
||||
return vec_xl(0, (const float *)(tmp));
|
||||
}
|
||||
template <> inline float32x4_t load(const ggml_bf16_t * p) {
|
||||
uint64_t bits;
|
||||
memcpy(&bits, p, sizeof(bits));
|
||||
const uint64x2_t v_bits = { bits, bits };
|
||||
return GGML_BF16_TO_F32_HI((GGML_BF16_VEC) v_bits);
|
||||
}
|
||||
template <> inline float32x4_t load(const float * p) {
|
||||
return vec_xl(0, p);
|
||||
}
|
||||
@@ -1302,6 +1308,88 @@ class tinyBLAS_RVV {
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// QUANT ZERO MATRIX MULTIPLICATION
|
||||
|
||||
#if defined(__VXE__) || defined(__VXE2__)
|
||||
template <typename TA>
|
||||
class tinyBLAS_Q0_VXE {
|
||||
static_assert(sizeof(TA::qs) == QK8_0, "VXE weight loads require 32 quantized bytes per block");
|
||||
|
||||
public:
|
||||
tinyBLAS_Q0_VXE(int64_t k,
|
||||
const TA * A, int64_t lda,
|
||||
const block_q8_0 * B, int64_t ldb,
|
||||
float * C, int64_t ldc,
|
||||
int32_t ith, int32_t nth)
|
||||
: A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
|
||||
}
|
||||
|
||||
void matmul(int64_t m, int64_t n) {
|
||||
constexpr int32_t tile_size = 2;
|
||||
const int64_t row_tiles = m / tile_size;
|
||||
const int64_t tiles = row_tiles * (n / tile_size);
|
||||
const int64_t duty = (tiles + nth - 1) / nth;
|
||||
const int64_t start = duty * ith;
|
||||
const int64_t end = MIN(start + duty, tiles);
|
||||
|
||||
for (int64_t job = start; job < end; ++job) {
|
||||
const int64_t ii = tile_size * (job % row_tiles);
|
||||
const int64_t jj = tile_size * (job / row_tiles);
|
||||
float32x4_t v_sum[tile_size][tile_size] = {};
|
||||
|
||||
for (int64_t l = 0; l < k; ++l) {
|
||||
int8x16_t v_xl[tile_size], v_xh[tile_size], v_yl[tile_size], v_yh[tile_size];
|
||||
float x_scale[tile_size], y_scale[tile_size];
|
||||
|
||||
for (int32_t i = 0; i < tile_size; ++i) {
|
||||
const TA & x = A[lda * (ii + i) + l];
|
||||
const block_q8_0 & y = B[ldb * (jj + i) + l];
|
||||
|
||||
const int8_t * x_qs = x.qs;
|
||||
const int8_t * y_qs = y.qs;
|
||||
v_xl[i] = vec_xl(0, x_qs);
|
||||
v_xh[i] = vec_xl(0, x_qs + QK8_0 / 2);
|
||||
v_yl[i] = vec_xl(0, y_qs);
|
||||
v_yh[i] = vec_xl(0, y_qs + QK8_0 / 2);
|
||||
x_scale[i] = unhalf(x.d);
|
||||
y_scale[i] = unhalf(y.d);
|
||||
}
|
||||
|
||||
for (int32_t j = 0; j < tile_size; ++j) {
|
||||
for (int32_t i = 0; i < tile_size; ++i) {
|
||||
const int32x4_t v_dot = dot_s8_wide(v_xl[i], v_yl[j]) + dot_s8_wide(v_xh[i], v_yh[j]);
|
||||
const float32x4_t v_scale = vec_splats(x_scale[i] * y_scale[j]);
|
||||
v_sum[j][i] = vec_madd(vec_float(v_dot), v_scale, v_sum[j][i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (int32_t j = 0; j < tile_size; ++j) {
|
||||
for (int32_t i = 0; i < tile_size; ++i) {
|
||||
C[ldc * (jj + j) + ii + i] = hsum(v_sum[j][i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
static int32x4_t dot_s8_wide(int8x16_t v_x, int8x16_t v_y) {
|
||||
const int16x8_t v_even = vec_mule(v_x, v_y);
|
||||
const int16x8_t v_odd = vec_mulo(v_x, v_y);
|
||||
// Two (-128) * (-128) products can overflow a signed 16-bit sum.
|
||||
return vec_unpackh(v_even) + vec_unpackl(v_even) + vec_unpackh(v_odd) + vec_unpackl(v_odd);
|
||||
}
|
||||
|
||||
const TA * const A;
|
||||
const block_q8_0 * const B;
|
||||
float * const C;
|
||||
const int64_t k;
|
||||
const int64_t lda;
|
||||
const int64_t ldb;
|
||||
const int64_t ldc;
|
||||
const int32_t ith;
|
||||
const int32_t nth;
|
||||
};
|
||||
#endif
|
||||
|
||||
#if defined(__ARM_FEATURE_DOTPROD)
|
||||
template <typename TA>
|
||||
class tinyBLAS_Q0_ARM {
|
||||
@@ -4006,6 +4094,14 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64
|
||||
(float *)C, ldc};
|
||||
return tb.matmul(m, n);
|
||||
}
|
||||
#elif defined(__VXE__) || defined(__VXE2__)
|
||||
if (Btype == GGML_TYPE_BF16) {
|
||||
tinyBLAS<4, float32x4_t, float32x4_t, ggml_bf16_t, ggml_bf16_t, float> tb{ params, k,
|
||||
(const ggml_bf16_t *)A, lda,
|
||||
(const ggml_bf16_t *)B, ldb,
|
||||
(float *)C, ldc};
|
||||
return tb.matmul(m, n);
|
||||
}
|
||||
#elif defined(__MMA__)
|
||||
if (k % 8) {
|
||||
return false;
|
||||
@@ -4162,6 +4258,17 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64
|
||||
params->ith, params->nth};
|
||||
tb.matmul(m, n);
|
||||
return true;
|
||||
#elif defined(__VXE__) || defined(__VXE2__)
|
||||
if (m % 2 || n % 2) {
|
||||
return false;
|
||||
}
|
||||
tinyBLAS_Q0_VXE<block_q8_0> tb{
|
||||
k, (const block_q8_0 *)A, lda,
|
||||
(const block_q8_0 *)B, ldb,
|
||||
(float *)C, ldc,
|
||||
params->ith, params->nth};
|
||||
tb.matmul(m, n);
|
||||
return true;
|
||||
#elif defined(__MMA__)
|
||||
//TO-DO: Remove this condition once gemv forwarding is enabled.
|
||||
if (n < 8 && n != 4)
|
||||
|
||||
Reference in new issue
Block a user