ggml-cpu: SVE 256 bit implementation of gemm_q6_K_8x8_q8_K kernel (#27472)
* q6_k GEMM working * cleanup * review comments * review comment addressed - removed pg16_8 * incorporated the review comments - replaced switch with if and other points * marking q8_k_blocklen UNUSEDto disable warning for non-i8mm cases
This commit is contained in:
1 parent
c02b5c641e
commit
ac8e1694d8
1 file changed
+188
-1
@@ -4928,6 +4928,7 @@ void ggml_gemm_q6_K_8x8_q8_K(int n,
|
||||
|
||||
constexpr int ncols_interleaved = 8;
|
||||
constexpr int blocklen = 8;
|
||||
constexpr int q8_k_blocklen = 4;
|
||||
|
||||
assert(n % qk == 0);
|
||||
assert(nr % 4 == 0);
|
||||
@@ -4936,9 +4937,195 @@ void ggml_gemm_q6_K_8x8_q8_K(int n,
|
||||
UNUSED(nb);
|
||||
UNUSED(ncols_interleaved);
|
||||
UNUSED(blocklen);
|
||||
UNUSED(q8_k_blocklen);
|
||||
|
||||
#if defined(__aarch64__) && defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8)
|
||||
if (svcntb() * 8 == 256) {
|
||||
const svbool_t pg8 = svptrue_b8();
|
||||
const svbool_t pg32 = svptrue_b32();
|
||||
|
||||
const svuint8_t low_nibble = svdup_n_u8(0x0f);
|
||||
const svuint8_t high_lo2 = svdup_n_u8(0x03);
|
||||
const svuint8_t high_hi2 = svdup_n_u8(0x30);
|
||||
const svint8_t q6_bias = svdup_n_s8(32);
|
||||
const svint32_t zero_s32 = svdup_n_s32(0);
|
||||
|
||||
for (int y = 0; y < nr / q8_k_blocklen; ++y) {
|
||||
const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + y * nb;
|
||||
|
||||
for (int x = 0; x < nc / ncols_interleaved; ++x) {
|
||||
const block_q6_Kx8 * GGML_RESTRICT q6_ptr = (const block_q6_Kx8 *) vx + x * nb;
|
||||
|
||||
// one accumlator per output row
|
||||
svfloat32_t acc_f32_r0 = svdup_n_f32(0);
|
||||
svfloat32_t acc_f32_r1 = svdup_n_f32(0);
|
||||
svfloat32_t acc_f32_r2 = svdup_n_f32(0);
|
||||
svfloat32_t acc_f32_r3 = svdup_n_f32(0);
|
||||
|
||||
for (int b = 0; b < nb; ++b) {
|
||||
svint32_t acc_c03_r01 = zero_s32;
|
||||
svint32_t acc_c03_r23 = zero_s32;
|
||||
svint32_t acc_c47_r01 = zero_s32;
|
||||
svint32_t acc_c47_r23 = zero_s32;
|
||||
|
||||
for (int half = 0; half < 2; ++half) {
|
||||
const uint8_t * ql_base = q6_ptr[b].ql + half * 512;
|
||||
const uint8_t * qh_base = q6_ptr[b].qh + half * 256;
|
||||
|
||||
for (int sb = 0; sb < QK_K / 64; ++sb) {
|
||||
const int8_t * q8_base_l = q8_ptr[b].qs + half * 512 + sb * 64;
|
||||
const int8_t * q8_base_h = q8_ptr[b].qs + half * 512 + 256 + sb * 64;
|
||||
|
||||
// load 16 bytes (8B of R0, 8B of R1) and repeat in upper half
|
||||
// low lanes will multiply with 16 bytes of C0 and high lanes will multiply with 16 bytes of C1
|
||||
const svint8_t q8_l_r01_k0 = svld1rq_s8(pg8, q8_base_l + 0);
|
||||
const svint8_t q8_l_r01_k1 = svld1rq_s8(pg8, q8_base_l + 32);
|
||||
const svint8_t q8_l_r23_k0 = svld1rq_s8(pg8, q8_base_l + 16);
|
||||
const svint8_t q8_l_r23_k1 = svld1rq_s8(pg8, q8_base_l + 48);
|
||||
|
||||
const svint8_t q8_h_r01_k0 = svld1rq_s8(pg8, q8_base_h + 0);
|
||||
const svint8_t q8_h_r01_k1 = svld1rq_s8(pg8, q8_base_h + 32);
|
||||
const svint8_t q8_h_r23_k0 = svld1rq_s8(pg8, q8_base_h + 16);
|
||||
const svint8_t q8_h_r23_k1 = svld1rq_s8(pg8, q8_base_h + 48);
|
||||
|
||||
const int ql_off = sb * QK_K / 2;
|
||||
const int qh_off = ql_off & 255;
|
||||
|
||||
const int scale_idx_l = half * 8 + sb;
|
||||
const int scale_idx_h = scale_idx_l + 4;
|
||||
|
||||
const svint32_t scales_l_all = svld1sb_s32(pg32, q6_ptr[b].scales + scale_idx_l * 8);
|
||||
const svint32_t scales_h_all = svld1sb_s32(pg32, q6_ptr[b].scales + scale_idx_h * 8);
|
||||
|
||||
const svint32_t scales_l_c03 = svzip1_s32(scales_l_all, scales_l_all);
|
||||
const svint32_t scales_h_c03 = svzip1_s32(scales_h_all, scales_h_all);
|
||||
|
||||
{
|
||||
svuint8_t q6_ql0_01 = svld1_u8(pg8, ql_base + ql_off);
|
||||
svuint8_t q6_ql1_01 = svld1_u8(pg8, ql_base + ql_off + 64);
|
||||
svuint8_t q6_qh0_01 = svld1_u8(pg8, qh_base + qh_off);
|
||||
svuint8_t q6_qh1_01 = svld1_u8(pg8, qh_base + qh_off + 64);
|
||||
|
||||
if (sb > 1) {
|
||||
q6_qh0_01 = svlsr_n_u8_x(pg8, q6_qh0_01, 2);
|
||||
q6_qh1_01 = svlsr_n_u8_x(pg8, q6_qh1_01, 2);
|
||||
}
|
||||
|
||||
const svuint8_t q6_qh0_01_hi = svand_u8_x(pg8, q6_qh0_01, high_hi2);
|
||||
const svuint8_t q6_qh1_01_hi = svand_u8_x(pg8, q6_qh1_01, high_hi2);
|
||||
|
||||
const svuint8_t q6_l0_01_bits = svorr_u8_x(pg8, svand_u8_x(pg8, q6_ql0_01, low_nibble), svlsl_n_u8_x(pg8, svand_u8_x(pg8, q6_qh0_01, high_lo2), 4));
|
||||
const svuint8_t q6_l1_01_bits = svorr_u8_x(pg8, svand_u8_x(pg8, q6_ql1_01, low_nibble), svlsl_n_u8_x(pg8, svand_u8_x(pg8, q6_qh1_01, high_lo2), 4));
|
||||
const svuint8_t q6_h0_01_bits = svorr_u8_x(pg8, svlsr_n_u8_x(pg8, q6_ql0_01, 4), q6_qh0_01_hi);
|
||||
const svuint8_t q6_h1_01_bits = svorr_u8_x(pg8, svlsr_n_u8_x(pg8, q6_ql1_01, 4), q6_qh1_01_hi);
|
||||
|
||||
const svint8_t q6_l0_01 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_l0_01_bits), q6_bias);
|
||||
const svint8_t q6_l1_01 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_l1_01_bits), q6_bias);
|
||||
const svint8_t q6_h0_01 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_h0_01_bits), q6_bias);
|
||||
const svint8_t q6_h1_01 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_h1_01_bits), q6_bias);
|
||||
|
||||
svint32_t dot_l_r01 = svmmla_s32(zero_s32, q6_l0_01, q8_l_r01_k0);
|
||||
dot_l_r01 = svmmla_s32(dot_l_r01, q6_l1_01, q8_l_r01_k1);
|
||||
|
||||
svint32_t dot_h_r01 = svmmla_s32(zero_s32, q6_h0_01, q8_h_r01_k0);
|
||||
dot_h_r01 = svmmla_s32(dot_h_r01, q6_h1_01, q8_h_r01_k1);
|
||||
|
||||
svint32_t dot_l_r23 = svmmla_s32(zero_s32, q6_l0_01, q8_l_r23_k0);
|
||||
dot_l_r23 = svmmla_s32(dot_l_r23, q6_l1_01, q8_l_r23_k1);
|
||||
|
||||
svint32_t dot_h_r23 = svmmla_s32(zero_s32, q6_h0_01, q8_h_r23_k0);
|
||||
dot_h_r23 = svmmla_s32(dot_h_r23, q6_h1_01, q8_h_r23_k1);
|
||||
|
||||
acc_c03_r01 = svmla_s32_x(pg32, acc_c03_r01, dot_l_r01, scales_l_c03);
|
||||
acc_c03_r01 = svmla_s32_x(pg32, acc_c03_r01, dot_h_r01, scales_h_c03);
|
||||
acc_c03_r23 = svmla_s32_x(pg32, acc_c03_r23, dot_l_r23, scales_l_c03);
|
||||
acc_c03_r23 = svmla_s32_x(pg32, acc_c03_r23, dot_h_r23, scales_h_c03);
|
||||
}
|
||||
|
||||
const svint32_t scales_l_c47 = svzip2_s32(scales_l_all, scales_l_all);
|
||||
const svint32_t scales_h_c47 = svzip2_s32(scales_h_all, scales_h_all);
|
||||
|
||||
{
|
||||
svuint8_t q6_ql0_23 = svld1_u8(pg8, ql_base + ql_off + 32);
|
||||
svuint8_t q6_ql1_23 = svld1_u8(pg8, ql_base + ql_off + 96);
|
||||
svuint8_t q6_qh0_23 = svld1_u8(pg8, qh_base + qh_off + 32);
|
||||
svuint8_t q6_qh1_23 = svld1_u8(pg8, qh_base + qh_off + 96);
|
||||
|
||||
if (sb > 1) {
|
||||
q6_qh0_23 = svlsr_n_u8_x(pg8, q6_qh0_23, 2);
|
||||
q6_qh1_23 = svlsr_n_u8_x(pg8, q6_qh1_23, 2);
|
||||
}
|
||||
|
||||
const svuint8_t q6_qh0_23_hi = svand_u8_x(pg8, q6_qh0_23, high_hi2);
|
||||
const svuint8_t q6_qh1_23_hi = svand_u8_x(pg8, q6_qh1_23, high_hi2);
|
||||
|
||||
const svuint8_t q6_l0_23_bits = svorr_u8_x(pg8, svand_u8_x(pg8, q6_ql0_23, low_nibble), svlsl_n_u8_x(pg8, svand_u8_x(pg8, q6_qh0_23, high_lo2), 4));
|
||||
const svuint8_t q6_l1_23_bits = svorr_u8_x(pg8, svand_u8_x(pg8, q6_ql1_23, low_nibble), svlsl_n_u8_x(pg8, svand_u8_x(pg8, q6_qh1_23, high_lo2), 4));
|
||||
const svuint8_t q6_h0_23_bits = svorr_u8_x(pg8, svlsr_n_u8_x(pg8, q6_ql0_23, 4), q6_qh0_23_hi);
|
||||
const svuint8_t q6_h1_23_bits = svorr_u8_x(pg8, svlsr_n_u8_x(pg8, q6_ql1_23, 4), q6_qh1_23_hi);
|
||||
|
||||
const svint8_t q6_l0_23 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_l0_23_bits), q6_bias);
|
||||
const svint8_t q6_l1_23 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_l1_23_bits), q6_bias);
|
||||
const svint8_t q6_h0_23 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_h0_23_bits), q6_bias);
|
||||
const svint8_t q6_h1_23 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_h1_23_bits), q6_bias);
|
||||
|
||||
svint32_t dot_l_r01 = svmmla_s32(zero_s32, q6_l0_23, q8_l_r01_k0);
|
||||
dot_l_r01 = svmmla_s32(dot_l_r01, q6_l1_23, q8_l_r01_k1);
|
||||
|
||||
svint32_t dot_h_r01 = svmmla_s32(zero_s32, q6_h0_23, q8_h_r01_k0);
|
||||
dot_h_r01 = svmmla_s32(dot_h_r01, q6_h1_23, q8_h_r01_k1);
|
||||
|
||||
svint32_t dot_l_r23 = svmmla_s32(zero_s32, q6_l0_23, q8_l_r23_k0);
|
||||
dot_l_r23 = svmmla_s32(dot_l_r23, q6_l1_23, q8_l_r23_k1);
|
||||
|
||||
svint32_t dot_h_r23 = svmmla_s32(zero_s32, q6_h0_23, q8_h_r23_k0);
|
||||
dot_h_r23 = svmmla_s32(dot_h_r23, q6_h1_23, q8_h_r23_k1);
|
||||
|
||||
acc_c47_r01 = svmla_s32_x(pg32, acc_c47_r01, dot_l_r01, scales_l_c47);
|
||||
acc_c47_r01 = svmla_s32_x(pg32, acc_c47_r01, dot_h_r01, scales_h_c47);
|
||||
acc_c47_r23 = svmla_s32_x(pg32, acc_c47_r23, dot_l_r23, scales_l_c47);
|
||||
acc_c47_r23 = svmla_s32_x(pg32, acc_c47_r23, dot_h_r23, scales_h_c47);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// acc_c03_r01 -> c0-r0,c0-r1,c1-r0,c1-r1,c2-r0,c2-r1,c3-r0,c3-r1
|
||||
// acc_c47_r01 -> c4-r0,c4-r1,c5-r0,c5-r1,c6-r0,c6-r1,c7-r0,c7-r1
|
||||
// svuzp1_s32 picks even lanes -> c0-r0,c1-r0,c2-r0,c3-r0,c4-r0,c5-r0,c6-r0,c7-r0
|
||||
// similarly svuzp2_s32 picks remaining values from odd lanes
|
||||
const svint32_t dot_r0 = svuzp1_s32(acc_c03_r01, acc_c47_r01);
|
||||
const svint32_t dot_r1 = svuzp2_s32(acc_c03_r01, acc_c47_r01);
|
||||
const svint32_t dot_r2 = svuzp1_s32(acc_c03_r23, acc_c47_r23);
|
||||
const svint32_t dot_r3 = svuzp2_s32(acc_c03_r23, acc_c47_r23);
|
||||
|
||||
const svuint32_t q6_d_bits = svld1uh_u32(pg32, (const uint16_t *) q6_ptr[b].d);
|
||||
const svfloat32_t q6_d_f32 = svcvt_f32_f16_x(pg32, svreinterpret_f16_u32(q6_d_bits));
|
||||
|
||||
const svfloat32_t scale_r0 = svmul_n_f32_x(pg32, q6_d_f32, q8_ptr[b].d[0]);
|
||||
const svfloat32_t scale_r1 = svmul_n_f32_x(pg32, q6_d_f32, q8_ptr[b].d[1]);
|
||||
const svfloat32_t scale_r2 = svmul_n_f32_x(pg32, q6_d_f32, q8_ptr[b].d[2]);
|
||||
const svfloat32_t scale_r3 = svmul_n_f32_x(pg32, q6_d_f32, q8_ptr[b].d[3]);
|
||||
|
||||
acc_f32_r0 = svmla_f32_x(pg32, acc_f32_r0, svcvt_f32_s32_x(pg32, dot_r0), scale_r0);
|
||||
acc_f32_r1 = svmla_f32_x(pg32, acc_f32_r1, svcvt_f32_s32_x(pg32, dot_r1), scale_r1);
|
||||
acc_f32_r2 = svmla_f32_x(pg32, acc_f32_r2, svcvt_f32_s32_x(pg32, dot_r2), scale_r2);
|
||||
acc_f32_r3 = svmla_f32_x(pg32, acc_f32_r3, svcvt_f32_s32_x(pg32, dot_r3), scale_r3);
|
||||
}
|
||||
|
||||
const int row = y * q8_k_blocklen;
|
||||
const int col = x * ncols_interleaved;
|
||||
|
||||
svst1_f32(pg32, s + (row + 0) * bs + col, acc_f32_r0);
|
||||
svst1_f32(pg32, s + (row + 1) * bs + col, acc_f32_r1);
|
||||
svst1_f32(pg32, s + (row + 2) * bs + col, acc_f32_r2);
|
||||
svst1_f32(pg32, s + (row + 3) * bs + col, acc_f32_r3);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
#endif // SVE compile-time end
|
||||
|
||||
#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
|
||||
constexpr int q8_k_blocklen = 4;
|
||||
const uint8x16_t m4b = vdupq_n_u8(0x0f);
|
||||
const uint8x16_t mask_lo = vdupq_n_u8(0x03);
|
||||
const uint8x16_t mask_hi = vdupq_n_u8(0x30);
|
||||
|
||||
Reference in new issue
Block a user