diff --git a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h index 698d6a33b..40404bf10 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +++ b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h @@ -631,17 +631,40 @@ static void dequantize_tiled_weight_to_fp16_task_q6_k( HVX_Vector v_scale_k16 = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc_k16, v_sc_k16, -2)); #pragma unroll - for (int g = 0; g < 8; g++) { + for (int g = 0; g < 8; g += 4) { const HVX_Vector v_scale = (g < 4) ? v_scale_k0 : v_scale_k16; - HVX_Vector v_q = unpack_q6_k_group(vptr, g, mask_0f, mask_03, i32); - HVX_VectorPair vp16 = Q6_Wh_vunpack_Vb(v_q); - HVX_VectorPair vp_k = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16), Q6_V_lo_W(vp16), -4); + HVX_Vector v_q0 = unpack_q6_k_group(vptr, g + 0, mask_0f, mask_03, i32); + HVX_Vector v_q1 = unpack_q6_k_group(vptr, g + 1, mask_0f, mask_03, i32); + HVX_Vector v_q2 = unpack_q6_k_group(vptr, g + 2, mask_0f, mask_03, i32); + HVX_Vector v_q3 = unpack_q6_k_group(vptr, g + 3, mask_0f, mask_03, i32); - hvx_vmem(dst_ptr + (2 * g + 0) * 64) = - Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k)), v_scale)); - hvx_vmem(dst_ptr + (2 * g + 1) * 64) = - Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k)), v_scale)); + HVX_VectorPair vp16_0 = Q6_Wh_vunpack_Vb(v_q0); + HVX_VectorPair vp16_1 = Q6_Wh_vunpack_Vb(v_q1); + HVX_VectorPair vp16_2 = Q6_Wh_vunpack_Vb(v_q2); + HVX_VectorPair vp16_3 = Q6_Wh_vunpack_Vb(v_q3); + HVX_VectorPair vp_k0 = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16_0), Q6_V_lo_W(vp16_0), -4); + HVX_VectorPair vp_k1 = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16_1), Q6_V_lo_W(vp16_1), -4); + HVX_VectorPair vp_k2 = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16_2), Q6_V_lo_W(vp16_2), -4); + HVX_VectorPair vp_k3 = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16_3), Q6_V_lo_W(vp16_3), -4); + + HVX_Vector v_out00 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k0)), v_scale)); + HVX_Vector v_out01 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k0)), v_scale)); + HVX_Vector v_out10 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k1)), v_scale)); + HVX_Vector v_out11 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k1)), v_scale)); + HVX_Vector v_out20 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k2)), v_scale)); + HVX_Vector v_out21 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k2)), v_scale)); + HVX_Vector v_out30 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k3)), v_scale)); + HVX_Vector v_out31 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k3)), v_scale)); + + hvx_vmem(dst_ptr + (2 * g + 0) * 64) = v_out00; + hvx_vmem(dst_ptr + (2 * g + 1) * 64) = v_out01; + hvx_vmem(dst_ptr + (2 * g + 2) * 64) = v_out10; + hvx_vmem(dst_ptr + (2 * g + 3) * 64) = v_out11; + hvx_vmem(dst_ptr + (2 * g + 4) * 64) = v_out20; + hvx_vmem(dst_ptr + (2 * g + 5) * 64) = v_out21; + hvx_vmem(dst_ptr + (2 * g + 6) * 64) = v_out30; + hvx_vmem(dst_ptr + (2 * g + 7) * 64) = v_out31; } } }