diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index b214b1543..533f51a0e 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -288,8 +288,8 @@ static inline bool ggml_hexagon_is_repack_type(enum ggml_type type) { type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q2_K; } -// Size of one repacked row in the DSP tiled layout. The K-quant tiles store uncompressed scales/mins, -// so they are larger than the ggml blocks. For the other repack types the tile has the same size as the ggml blocks. +// Size of one repacked row in the DSP tiled layout. The Q3_K..Q6_K tiles store uncompressed scales/mins, +// so they are larger than the ggml blocks. For the other repack types (Q2_K included) the tile has the same size. static inline size_t ggml_hexagon_tiled_row_size(enum ggml_type type, int64_t ne0) { if (type == GGML_TYPE_Q6_K) { return (size_t) (ne0 / 32) * (HTP_MM_WEIGHT_TILE_SIZE_Q6_K / 32); @@ -303,9 +303,6 @@ static inline size_t ggml_hexagon_tiled_row_size(enum ggml_type type, int64_t ne if (type == GGML_TYPE_Q3_K) { return (size_t) (ne0 / 32) * (HTP_MM_WEIGHT_TILE_SIZE_Q3_K / 32); } - if (type == GGML_TYPE_Q2_K) { - return (size_t) (ne0 / 32) * (HTP_MM_WEIGHT_TILE_SIZE_Q2_K / 32); - } return ggml_row_size(type, ne0); } @@ -1909,13 +1906,11 @@ static void repack_q2_K_tiled(ggml_tensor * t, const void * data, size_t offset, for (int kt = 0; kt < n_k_tiles; kt++) { const int kt_local = kt % 8; // k-tile within the super-block const block_q2_K * b = &src_row[kt / 8]; - const float d = GGML_FP16_TO_FP32(b->d); - const float dmin = GGML_FP16_TO_FP32(b->dmin); uint8_t * tile = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size; uint8_t * lo_pl = tile; - ggml_half * sc_pl = (ggml_half *) (tile + 256); - ggml_half * m_pl = (ggml_half *) (tile + 384); + uint8_t * sc_pl = tile + 256; + ggml_half * dd_pl = (ggml_half *) (tile + 320); for (int lk = 0; lk < 32; lk++) { const int g = lk >> 2; @@ -1923,9 +1918,10 @@ static void repack_q2_K_tiled(ggml_tensor * t, const void * data, size_t offset, lo_pl[(g >> 2) * 128 + pos] |= (uint8_t) (q2_3_K_get_low2(b->qs, kt_local * 32 + lk) << ((g & 3) * 2)); } for (int sub = 0; sub < 2; sub++) { - const uint8_t sc = b->scales[kt_local * 2 + sub]; - sc_pl[sub * 32 + row] = GGML_FP32_TO_FP16( d * (float) (sc & 0xF)); - m_pl [sub * 32 + row] = GGML_FP32_TO_FP16(-dmin * (float) (sc >> 4)); + sc_pl[sub * 32 + row] = b->scales[kt_local * 2 + sub]; + } + if (row / 8 == kt_local % 4) { + dd_pl[row % 8] = kt_local < 4 ? b->d : (ggml_half) (b->dmin ^ 0x8000); // -dmin } } } @@ -1935,7 +1931,7 @@ static void repack_q2_K_tiled(ggml_tensor * t, const void * data, size_t offset, GGML_UNUSED(size); } -// Reverse of repack_q2_K_tiled. Unpacks quants losslessly and normalizes scales/mins. Read-back only. +// Reverse of repack_q2_K_tiled. Lossless. Read-back only. static void repack_tiled_q2_K(void * data, const ggml_tensor * t, size_t offset, size_t size) { GGML_ASSERT(offset == 0); @@ -1970,14 +1966,12 @@ static void repack_tiled_q2_K(void * data, const ggml_tensor * t, size_t offset, block_q2_K * b = &dst_row[sb]; memset(b, 0, sizeof(block_q2_K)); - ggml_half sub_scales[16]; - ggml_half sub_mins[16]; for (int kt_local = 0; kt_local < 8; kt_local++) { const int kt = sb * 8 + kt_local; const uint8_t * tile = matrix_src + ((size_t) ct * n_k_tiles + kt) * tile_size; const uint8_t * lo_pl = tile; - const ggml_half * sc_pl = (const ggml_half *) (tile + 256); - const ggml_half * m_pl = (const ggml_half *) (tile + 384); + const uint8_t * sc_pl = tile + 256; + const ggml_half * dd_pl = (const ggml_half *) (tile + 320); for (int lk = 0; lk < 32; lk++) { const int e = kt_local * 32 + lk; @@ -1988,36 +1982,15 @@ static void repack_tiled_q2_K(void * data, const ggml_tensor * t, size_t offset, } for (int sub = 0; sub < 2; sub++) { - const float D = GGML_FP16_TO_FP32(sc_pl[sub * 32 + row]); - const float M = GGML_FP16_TO_FP32(m_pl[sub * 32 + row]); - sub_scales[kt_local * 2 + sub] = GGML_FP32_TO_FP16((D > 0.0f) ? D : 0.0f); - sub_mins[kt_local * 2 + sub] = GGML_FP32_TO_FP16((-M > 0.0f) ? -M : 0.0f); + b->scales[kt_local * 2 + sub] = sc_pl[sub * 32 + row]; } - } - - int ls[16]; - int lm[16]; - ggml_half * const dd[2] = { &b->d, &b->dmin }; - const ggml_half * const prod[2] = { sub_scales, sub_mins }; - int * const ll[2] = { ls, lm }; - for (int w = 0; w < 2; w++) { - if (hexagon_recover_k_scales(prod[w], 16, 0, 15, dd[w], ll[w])) { - continue; + if (row / 8 == kt_local % 4) { + if (kt_local < 4) { + b->d = dd_pl[row % 8]; + } else { + b->dmin = (ggml_half) (dd_pl[row % 8] ^ 0x8000); + } } - float max_val = 0.0f; - for (int j = 0; j < 16; j++) { - max_val = (std::max)(max_val, GGML_FP16_TO_FP32(prod[w][j])); - } - *dd[w] = GGML_FP32_TO_FP16(max_val / 15.0f); - const float d_actual = GGML_FP16_TO_FP32(*dd[w]); - const float inv_d = (d_actual > 0.0f) ? (1.0f / d_actual) : 0.0f; - for (int j = 0; j < 16; j++) { - ll[w][j] = (std::min)(15, (int) roundf(inv_d * GGML_FP16_TO_FP32(prod[w][j]))); - } - } - - for (int j = 0; j < 16; j++) { - b->scales[j] = (uint8_t) (ls[j] | (lm[j] << 4)); } } } diff --git a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h index 40404bf10..f0a0d9de1 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +++ b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h @@ -701,20 +701,29 @@ static void dequantize_tiled_weight_to_fp16_task_q3_k( } } -// Q2_K stores 2-bit weights and one fp16 scale and offset per 16 k, see HTP_MM_WEIGHT_TILE_SIZE_Q2_K. +// Q2_K stores 2-bit weights, the ggml scale bytes and d / -dmin per super-block, see HTP_MM_WEIGHT_TILE_SIZE_Q2_K. static void dequantize_tiled_weight_to_fp16_task_q2_k( const tiled_dequantize_state_t *state, uint32_t start_tile, uint32_t end_tile) { const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + HVX_Vector v_d = Q6_V_vzero(); + HVX_Vector v_dm = Q6_V_vzero(); + for (uint32_t t = start_tile; t < end_tile; t++) { - const HVX_Vector * vptr = (const HVX_Vector *) (state->src + t * state->aligned_tile_size); + const uint8_t * tile = state->src + t * state->aligned_tile_size; + const HVX_Vector * vptr = (const HVX_Vector *) tile; __fp16 * dst_ptr = state->dst + t * HTP_MM_HMX_TILE_N_ELMS; - HVX_Vector v_sc = vptr[2]; + // rows hold all k tiles and n_k_tiles is a multiple of 8, so t % 8 is the tile within its super-block + if (t == start_tile || (t & 7) == 0) { + q2_k_load_dmin(tile - (t & 7) * state->aligned_tile_size, &v_d, &v_dm); + } + + HVX_Vector v_sc, v_m; + q2_k_unpack_scales(tile, v_d, v_dm, &v_sc, &v_m); HVX_Vector v_sc_k16 = Q6_V_vror_VR(v_sc, 64); - HVX_Vector v_m = vptr[3]; HVX_Vector v_m_k16 = Q6_V_vror_VR(v_m, 64); HVX_Vector v_scale_k0 = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2)); HVX_Vector v_scale_k16 = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc_k16, v_sc_k16, -2)); diff --git a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h index 580bae4ac..0c86e407e 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +++ b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h @@ -1,19 +1,31 @@ // Dynamic quantizers that produce tiled activations -// vector 9: d * sum(q) of the 32 k, replicated (q8_1), or per 16 k for Q2_K (q8_1_s16): k 0..15 in lanes 0..31, -// k 16..31 in lanes 32..63, with d the fp16 scale of vector 8 +// q8_1: vector 9 holds d * sum(q) of the 32 k, replicated. +// q8_1_s16 (Q2_K, sums16): one fp16 d (vector 8) for the 256 k super-block starting at x_sb, and vector 9 holds the +// integer sums of k 0..15 (low halfword) and k 16..31 (high halfword) in every word __attribute__((always_inline)) -static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_t * restrict y_block, const bool sums16) { +static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_t * restrict y_block, const bool sums16, + const float * restrict x_sb) { assert((unsigned long) x % 128 == 0); assert((unsigned long) y_block % 128 == 0); HVX_Vector * vx = (HVX_Vector *) x; HVX_Vector zero = Q6_V_vzero(); - HVX_Vector vmax0_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[0])); - HVX_Vector vmax1_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[1])); - HVX_Vector vmax2_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[2])); - HVX_Vector vmax3_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[3])); + HVX_Vector vmax0_sf, vmax1_sf, vmax2_sf, vmax3_sf; + if (sums16) { + const HVX_Vector * vsb = (const HVX_Vector *) x_sb; + HVX_Vector vm = hvx_vec_abs_f32(vsb[0]); + for (int i = 1; i < 8; i++) { + vm = Q6_Vsf_vmax_VsfVsf(vm, hvx_vec_abs_f32(vsb[i])); + } + vmax0_sf = vmax1_sf = vmax2_sf = vmax3_sf = hvx_vec_reduce_max_f32(vm); + } else { + vmax0_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[0])); + vmax1_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[1])); + vmax2_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[2])); + vmax3_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[3])); + } HVX_Vector vx0_qf = Q6_Vqf32_vsub_VsfVsf(vx[0], zero); HVX_Vector vx1_qf = Q6_Vqf32_vsub_VsfVsf(vx[1], zero); @@ -61,15 +73,6 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_ HVX_Vector vd3_sf = hvx_vec_mul_f32_f32(vmax3_sf, v_inv127); HVX_Vector v_sums_sf = Q6_Vsf_equals_Vw(v_sums); - if (sums16) { - // the fp16 d of vector 8 - HVX_VectorPair vd01_sf = hvx_vec_f16_to_f32(vd01_hf); - HVX_VectorPair vd23_sf = hvx_vec_f16_to_f32(vd23_hf); - vd0_sf = Q6_V_lo_W(vd01_sf); - vd1_sf = Q6_V_hi_W(vd01_sf); - vd2_sf = Q6_V_lo_W(vd23_sf); - vd3_sf = Q6_V_hi_W(vd23_sf); - } HVX_Vector voff0_sf = hvx_vec_mul_f32_f32(vd0_sf, v_sums_sf); HVX_Vector voff1_sf = hvx_vec_mul_f32_f32(vd1_sf, Q6_V_vror_VR(v_sums_sf, 32)); HVX_Vector voff2_sf = hvx_vec_mul_f32_f32(vd2_sf, Q6_V_vror_VR(v_sums_sf, 64)); @@ -91,12 +94,12 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_ hvx_vec_repl_f16(Q6_V_vror_VR(voff23_hf, 64)), }; if (sums16) { - // the k 16..31 sums sit 4 halfwords after the k 0..15 sums - const HVX_VectorPred q_lo = Q6_Q_vsetq_R(64); - r_offset[0] = Q6_V_vmux_QVV(q_lo, r_offset[0], hvx_vec_repl_f16(Q6_V_vror_VR(voff01_hf, 8))); - r_offset[1] = Q6_V_vmux_QVV(q_lo, r_offset[1], hvx_vec_repl_f16(Q6_V_vror_VR(voff01_hf, 72))); - r_offset[2] = Q6_V_vmux_QVV(q_lo, r_offset[2], hvx_vec_repl_f16(Q6_V_vror_VR(voff23_hf, 8))); - r_offset[3] = Q6_V_vmux_QVV(q_lo, r_offset[3], hvx_vec_repl_f16(Q6_V_vror_VR(voff23_hf, 72))); + // |sum of 16 int8| <= 2032 fits a halfword + int32_t __attribute__((aligned(128))) ws[32]; + *(HVX_Vector *) ws = v_sums; + for (int b = 0; b < 4; b++) { + r_offset[b] = Q6_V_vsplat_R(((uint32_t) ws[8 * b + 4] << 16) | ((uint32_t) ws[8 * b] & 0xFFFF)); + } } static const uint8_t __attribute__((aligned(128))) repl[128] = { @@ -138,11 +141,11 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_ } static inline void quantize_block_f32_q8_1_tiled(float * restrict x, uint8_t * restrict y_block) { - quantize_block_f32_q8_1_tiled_impl(x, y_block, false); + quantize_block_f32_q8_1_tiled_impl(x, y_block, false, NULL); } -static inline void quantize_block_f32_q8_1_s16_tiled(float * restrict x, uint8_t * restrict y_block) { - quantize_block_f32_q8_1_tiled_impl(x, y_block, true); +static inline void quantize_block_f32_q8_1_s16_tiled(float * restrict x, uint8_t * restrict y_block, const float * restrict x_sb) { + quantize_block_f32_q8_1_tiled_impl(x, y_block, true, x_sb); } static inline void quantize_block_f32_q8_0_tiled(float * restrict x, uint8_t * restrict y_block) { @@ -267,7 +270,7 @@ static void quantize_row_f32_q8_1_s16_tiled(float * restrict x, uint8_t * restri for (uint32_t i = 0; i < nb; i++) { uint8_t * restrict y_block = y + i * 4 * 1280; - quantize_block_f32_q8_1_s16_tiled(x + i * qk, y_block); + quantize_block_f32_q8_1_s16_tiled(x + i * qk, y_block, x + (i & ~1u) * qk); // k % 256 == 0 for Q2_K } } @@ -655,7 +658,7 @@ static inline void accum_q3_k_32x2( *v_sums1 = Q6_W_vcombine_VV(v_sum1_hi, v_sum1_lo); } -// Q2_K (x = D * q + M): the Q3_K dot products without the -4 flags, M uses the q8_1_s16 sums +// Q2_K: the Q3_K dot products without the -4 flags, scales and mins are applied in q2_k_int_accum static inline HVX_VectorPair accum_q2_k_32x1( const HVX_Vector * restrict vptr, const HVX_Vector * restrict v_act @@ -702,11 +705,57 @@ static inline void accum_q2_k_32x2( *v_sums1 = Q6_W_vcombine_VV(v_sum1_hi, v_sum1_lo); } -// D as the Q6_K scales, plus M (vector 3) times the per-16 activation sums in v_act[9] -static inline HVX_Vector scale_q2_k_32x1(HVX_VectorPair v_sums, const HVX_Vector * restrict vptr, const HVX_Vector * restrict v_act) { - HVX_VectorPair v_m = hvx_vec_mul_f16_f16_to_f32_pair(vptr[3], v_act[9]); - HVX_Vector v_d = scale_q6_k_32x1(v_sums, vptr[2], v_act[8]); - return hvx_vec_add_f32_f32(v_d, hvx_vec_add_f32_f32(Q6_V_lo_W(v_m), Q6_V_hi_W(v_m))); +// Q2_K: fp16 d (*v_d) and -dmin (*v_m) of the 32 rows of a super-block (8 tiles, VTCM stride 384), each in both vector halves +static inline void q2_k_load_dmin(const uint8_t * restrict sb, HVX_Vector * restrict v_d, HVX_Vector * restrict v_m) { + HVX_Vector v_dd = Q6_V_vzero(); // lanes 0..31: d, lanes 32..63: -dmin + + #pragma unroll + for (int j = 0; j < 8; j++) { + HVX_Vector v = hvx_vmem(sb + j * 384 + 256); // the slice is at bytes 64..79 + v_dd = Q6_V_vmux_QVV(Q6_Q_vsetq_R(16 * j), v_dd, Q6_V_vror_VR(v, (64 - 16 * j) & 127)); + } + + HVX_VectorPred q_lo = Q6_Q_vsetq_R(64); + HVX_Vector v_r = Q6_V_vror_VR(v_dd, 64); + *v_d = Q6_V_vmux_QVV(q_lo, v_dd, v_r); + *v_m = Q6_V_vmux_QVV(q_lo, v_r, v_dd); +} + +// Q2_K: fp16 D = d * scale and M = -dmin * min of a tile from its scale bytes, in the lanes of the Q6_K scales +static inline void q2_k_unpack_scales(const uint8_t * restrict tile, HVX_Vector v_d, HVX_Vector v_m, + HVX_Vector * restrict v_D, HVX_Vector * restrict v_M) { + HVX_Vector v_sc = hvx_vmem(tile + 256); + HVX_Vector v_l = Q6_V_lo_W(Q6_Wuh_vunpack_Vub(Q6_V_vand_VV(v_sc, Q6_Vb_vsplat_R(0x0F)))); + HVX_Vector v_mn = Q6_V_lo_W(Q6_Wuh_vunpack_Vub(Q6_Vub_vlsr_VubR(v_sc, 4))); + // qf32 product: rounds like GGML_FP32_TO_FP16(d * scale) (qf16 is off by 1 ulp for many values on v73) + *v_D = Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(Q6_Vhf_equals_Vh(v_l), v_d)); + *v_M = Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(Q6_Vhf_equals_Vh(v_mn), v_m)); +} + +// Q2_K scale and min nibbles of a tile as halfword pairs: word r = sub-block 1 << 16 | sub-block 0 of row r +static inline void q2_k_scale_pairs(const uint8_t * restrict tile, HVX_Vector * restrict v_sc2, HVX_Vector * restrict v_mn2) { + HVX_Vector v_h = Q6_Vh_vshuff_Vh(Q6_V_lo_W(Q6_Wuh_vunpack_Vub(hvx_vmem(tile + 256)))); + *v_sc2 = Q6_V_vand_VV(v_h, Q6_Vh_vsplat_R(0x000F)); + *v_mn2 = Q6_Vuh_vlsr_VuhR(v_h, 4); +} + +// Q2_K with q8_1_s16 activations, one tile: isum += scale * (q . a), msum += min * sum(a), per row, in int32. +// |sum of 16 q * a| <= 3 * 128 * 16 fits a halfword, so both sub-blocks go through one vdmpy (no saturation) +static inline void q2_k_int_accum(HVX_VectorPair v_sums, HVX_Vector v_sc2, HVX_Vector v_mn2, HVX_Vector v_bsum, + HVX_Vector * restrict v_isum, HVX_Vector * restrict v_msum) { + HVX_Vector v_s2 = Q6_Vh_vshuffe_VhVh(Q6_V_hi_W(v_sums), Q6_V_lo_W(v_sums)); + *v_isum = Q6_Vw_vadd_VwVw(*v_isum, Q6_Vw_vdmpy_VhVh_sat(v_s2, v_sc2)); + *v_msum = Q6_Vw_vadd_VwVw(*v_msum, Q6_Vw_vdmpy_VhVh_sat(v_mn2, v_bsum)); +} + +// Q2_K super-block result: d_a * (d * isum - dmin * msum), with v_d / v_m the fp16 d / -dmin of q2_k_load_dmin +static inline HVX_Vector q2_k_int_scale(HVX_Vector v_isum, HVX_Vector v_msum, HVX_Vector v_d, HVX_Vector v_m, HVX_Vector v_act8) { + HVX_Vector v_d32 = Q6_V_lo_W(hvx_vec_f16_to_f32(v_d)); + HVX_Vector v_m32 = Q6_V_lo_W(hvx_vec_f16_to_f32(v_m)); + HVX_Vector v_da32 = Q6_V_lo_W(hvx_vec_f16_to_f32(v_act8)); + HVX_Vector v_t = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(v_d32, Q6_Vsf_equals_Vw(v_isum)), + hvx_vec_mul_f32_f32(v_m32, Q6_Vsf_equals_Vw(v_msum))); + return hvx_vec_mul_f32_f32(v_t, v_da32); } static void tiled_vec_dot_q4_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { @@ -1174,12 +1223,22 @@ static void tiled_vec_dot_q2_k_32x1(const uint32_t n, float * restrict s, const HVX_Vector v_sum_float = Q6_V_vzero(); uint32_t n_k_tiles = n / 32; - for (uint32_t kt = 0; kt < n_k_tiles; kt++) { - const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 512); - const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + kt * 1280); + for (uint32_t kb = 0; kb < n_k_tiles; kb += 8) { + HVX_Vector v_d, v_m; + q2_k_load_dmin(tile_ptr + kb * 384, &v_d, &v_m); - HVX_VectorPair v_sums = accum_q2_k_32x1(vptr, v_act); - v_sum_float = hvx_vec_add_f32_f32(v_sum_float, scale_q2_k_32x1(v_sums, vptr, v_act)); + HVX_Vector v_isum = Q6_V_vzero(); + HVX_Vector v_msum = Q6_V_vzero(); + for (uint32_t kt = kb; kt < kb + 8; kt++) { + const uint8_t * restrict tile = tile_ptr + kt * 384; + const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + kt * 1280); + + HVX_Vector v_sc2, v_mn2; + q2_k_scale_pairs(tile, &v_sc2, &v_mn2); + q2_k_int_accum(accum_q2_k_32x1((const HVX_Vector *) tile, v_act), v_sc2, v_mn2, v_act[9], &v_isum, &v_msum); + } + const HVX_Vector * restrict v_act_sb = (const HVX_Vector *) (y_q + kb * 1280); + v_sum_float = hvx_vec_add_f32_f32(v_sum_float, q2_k_int_scale(v_isum, v_msum, v_d, v_m, v_act_sb[8])); } if (sz) { @@ -1198,16 +1257,31 @@ static void tiled_vec_dot_q2_k_32x2(const uint32_t n, float * restrict s0, float HVX_Vector v_sum_float_c1 = Q6_V_vzero(); uint32_t n_k_tiles = n / 32; - for (uint32_t kt = 0; kt < n_k_tiles; kt++) { - const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 512); - const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1280); - const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1280); + for (uint32_t kb = 0; kb < n_k_tiles; kb += 8) { + HVX_Vector v_d, v_m; + q2_k_load_dmin(tile_ptr + kb * 384, &v_d, &v_m); - HVX_VectorPair v_sums0, v_sums1; - accum_q2_k_32x2(vptr, v_act0, v_act1, &v_sums0, &v_sums1); + HVX_Vector v_isum0 = Q6_V_vzero(); + HVX_Vector v_msum0 = Q6_V_vzero(); + HVX_Vector v_isum1 = Q6_V_vzero(); + HVX_Vector v_msum1 = Q6_V_vzero(); + for (uint32_t kt = kb; kt < kb + 8; kt++) { + const uint8_t * restrict tile = tile_ptr + kt * 384; + const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1280); + const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1280); - v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, scale_q2_k_32x1(v_sums0, vptr, v_act0)); - v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, scale_q2_k_32x1(v_sums1, vptr, v_act1)); + HVX_Vector v_sc2, v_mn2; + q2_k_scale_pairs(tile, &v_sc2, &v_mn2); + + HVX_VectorPair v_sums0, v_sums1; + accum_q2_k_32x2((const HVX_Vector *) tile, v_act0, v_act1, &v_sums0, &v_sums1); + q2_k_int_accum(v_sums0, v_sc2, v_mn2, v_act0[9], &v_isum0, &v_msum0); + q2_k_int_accum(v_sums1, v_sc2, v_mn2, v_act1[9], &v_isum1, &v_msum1); + } + const HVX_Vector * restrict v_act0_sb = (const HVX_Vector *) (y0_q + kb * 1280); + const HVX_Vector * restrict v_act1_sb = (const HVX_Vector *) (y1_q + kb * 1280); + v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, q2_k_int_scale(v_isum0, v_msum0, v_d, v_m, v_act0_sb[8])); + v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, q2_k_int_scale(v_isum1, v_msum1, v_d, v_m, v_act1_sb[8])); } if (sz0) { @@ -1539,7 +1613,7 @@ static inline void quantize_f32_q8_1_s16_tiled_block_kernel( const float * restrict src_ptr = (const float *) ((const uint8_t *) src + r * src_row_size + c * qk * sizeof(float)); uint8_t * restrict dst_ptr = dst + r * dst_row_size + c * 4 * 1280; - quantize_block_f32_q8_1_s16_tiled((float *) src_ptr, dst_ptr); + quantize_block_f32_q8_1_s16_tiled((float *) src_ptr, dst_ptr, src_ptr - (c & 1) * qk); // super-block of 256 k c++; if (c == nb) { diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.c index 1a513d160..28aed4680 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c @@ -543,7 +543,7 @@ MATMUL_2D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x2, tiled_vec_do MATMUL_2D_REPACKED_IMPL(q6_k, 896, tiled_vec_dot_q6_k_32x2, tiled_vec_dot_q6_k_32x1) MATMUL_2D_REPACKED_IMPL(q5_k, 768, tiled_vec_dot_q5_k_32x2, tiled_vec_dot_q5_k_32x1) MATMUL_2D_REPACKED_IMPL(q3_k, 512, tiled_vec_dot_q3_k_32x2, tiled_vec_dot_q3_k_32x1) -MATMUL_2D_REPACKED_IMPL(q2_k, 512, tiled_vec_dot_q2_k_32x2, tiled_vec_dot_q2_k_32x1) +MATMUL_2D_REPACKED_IMPL(q2_k, 336, tiled_vec_dot_q2_k_32x2, tiled_vec_dot_q2_k_32x1) MATMUL_2D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x2, tiled_vec_dot_iq4nl_32x1) MATMUL_2D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x2, tiled_vec_dot_mxfp4_32x1) @@ -761,7 +761,7 @@ MATVEC_2D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x1) MATVEC_2D_REPACKED_IMPL(q5_k, 768, tiled_vec_dot_q5_k_32x1) MATVEC_2D_REPACKED_IMPL(q6_k, 896, tiled_vec_dot_q6_k_32x1) MATVEC_2D_REPACKED_IMPL(q3_k, 512, tiled_vec_dot_q3_k_32x1) -MATVEC_2D_REPACKED_IMPL(q2_k, 512, tiled_vec_dot_q2_k_32x1) +MATVEC_2D_REPACKED_IMPL(q2_k, 336, tiled_vec_dot_q2_k_32x1) MATVEC_2D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x1) MATVEC_2D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x1) @@ -772,7 +772,7 @@ MATMUL_NX_2D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x2, tiled_vec_do MATMUL_NX_2D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x2, tiled_vec_dot_mxfp4_32x1) MATMUL_NX_2D_REPACKED_IMPL(q5_k, 768, tiled_vec_dot_q5_k_32x2, tiled_vec_dot_q5_k_32x1) MATMUL_NX_2D_REPACKED_IMPL(q3_k, 512, tiled_vec_dot_q3_k_32x2, tiled_vec_dot_q3_k_32x1) -MATMUL_NX_2D_REPACKED_IMPL(q2_k, 512, tiled_vec_dot_q2_k_32x2, tiled_vec_dot_q2_k_32x1) +MATMUL_NX_2D_REPACKED_IMPL(q2_k, 336, tiled_vec_dot_q2_k_32x2, tiled_vec_dot_q2_k_32x1) #define MATMUL_4D_REPACKED_IMPL(SUFFIX, TILE_SIZE, DOT_2X2, DOT_2X1) \ static void hvx_mm_4d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ @@ -907,7 +907,7 @@ MATMUL_4D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x2, tiled_vec_do MATMUL_4D_REPACKED_IMPL(q6_k, 896, tiled_vec_dot_q6_k_32x2, tiled_vec_dot_q6_k_32x1) MATMUL_4D_REPACKED_IMPL(q5_k, 768, tiled_vec_dot_q5_k_32x2, tiled_vec_dot_q5_k_32x1) MATMUL_4D_REPACKED_IMPL(q3_k, 512, tiled_vec_dot_q3_k_32x2, tiled_vec_dot_q3_k_32x1) -MATMUL_4D_REPACKED_IMPL(q2_k, 512, tiled_vec_dot_q2_k_32x2, tiled_vec_dot_q2_k_32x1) +MATMUL_4D_REPACKED_IMPL(q2_k, 336, tiled_vec_dot_q2_k_32x2, tiled_vec_dot_q2_k_32x1) MATMUL_4D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x2, tiled_vec_dot_iq4nl_32x1) MATMUL_4D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x2, tiled_vec_dot_mxfp4_32x1) diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.h b/ggml/src/ggml-hexagon/htp/matmul-ops.h index 1d11a4c12..522d8779f 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.h +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.h @@ -38,11 +38,11 @@ extern "C" { // vector 2: bit g set where the hmask bit of group g is clear (quant = low 2 bits - 4) // vector 3: fp16 scales per row, d * (scales[] - 32): k 0..15 in lanes 0..31, k 16..31 in lanes 32..63 #define HTP_MM_WEIGHT_TILE_SIZE_Q3_K 512 -// Q2_K native 2-bit tile, vrmpy-ready like Q6_K -// vectors 0..1: unsigned 2-bit quants, vector m holds groups 4m..4m+3 at bit offsets 0,2,4,6 -// vector 2: fp16 scales per row, d * (scales[] & 0xF), same lanes as Q3_K vector 3 -// vector 3: fp16 offsets per row, -dmin * (scales[] >> 4), same lanes -#define HTP_MM_WEIGHT_TILE_SIZE_Q2_K 512 +// Q2_K native 2-bit tile with the ggml scale bytes, vrmpy-ready like Q6_K (same bits per weight as block_q2_K) +// bytes 0..255: unsigned 2-bit quants, vector m holds groups 4m..4m+3 at bit offsets 0,2,4,6 +// bytes 256..319: scales[] bytes (min << 4 | scale), sub-block s of row r at 32 * s + r +// bytes 320..335: 8 fp16, tile j of a super-block: d (j < 4) or -dmin (j >= 4) of rows 8 * (j % 4) .. 8 * (j % 4) + 7 +#define HTP_MM_WEIGHT_TILE_SIZE_Q2_K 336 // --- Weight Repacked Aligned Tile Sizes --- #define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q4_0 640 @@ -53,7 +53,7 @@ extern "C" { #define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q5_K 768 #define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q6_K 896 #define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q3_K 512 -#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_K 512 +#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_K 384 // --- Activation Tiled Block Sizes (including padding) --- #define HTP_MM_ACT_TILE_SIZE_Q8_0 1152