From aa5e0092fdb8980b9e89498dfb73049b4fe204c2 Mon Sep 17 00:00:00 2001 From: kurquhar Date: Wed, 7 Oct 2026 17:04:45 -0700 Subject: [PATCH] hexagon: support tiled Q4_K and Q6_K GET_ROWS (#30115) * hexagon: support tiled Q4_K GET_ROWS Assisted-by: OpenCode * properly reject Q4_K views Assisted-by: OpenCode * hexagon: support tiled Q6_K GET_ROWS Assisted-by: OpenCode --- ggml/src/ggml-hexagon/ggml-hexagon.cpp | 16 +++--- ggml/src/ggml-hexagon/htp/get-rows-ops.c | 64 ++++++++++++++++++++---- ggml/src/ggml-hexagon/htp/get-rows-ops.h | 2 +- 3 files changed, 64 insertions(+), 18 deletions(-) diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index f0d768388..26b79582e 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -7232,7 +7232,7 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session * const struct ggml_tensor * src1 = op->src[1]; // indices const struct ggml_tensor * dst = op; - if (src0->type == GGML_TYPE_Q4_0 && src0->view_src) { + if ((src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q4_K || src0->type == GGML_TYPE_Q6_K) && src0->view_src) { return false; } @@ -7241,7 +7241,7 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session * if (src0_base->buffer && ggml_backend_buffer_is_hexagon(src0_base->buffer) && src0_base->extra) { const auto * extra = (const ggml_hexagon_tensor_extra *) src0_base->extra; is_repacked = (extra->flags & GGML_HEXAGON_TENSOR_REPACK) != 0; - if (is_repacked && src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0) { + if (is_repacked && src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q6_K && src0->type != GGML_TYPE_Q8_0) { return false; } } @@ -7252,7 +7252,7 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session * return false; } - if (src0->type == GGML_TYPE_Q4_0 && src0->buffer && !is_repacked) { + if ((src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q4_K || src0->type == GGML_TYPE_Q6_K) && src0->buffer && ggml_backend_buffer_get_size(src0->buffer) != 0 && !is_repacked) { return false; } @@ -7261,7 +7261,11 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session * } if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && - src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_I32) { + src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q6_K && src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_I32) { + return false; + } + + if ((src0->type == GGML_TYPE_Q4_K || src0->type == GGML_TYPE_Q6_K) && (!ggml_is_contiguous(src0) || ggml_is_permuted(src0) || src0->ne[0] % QK_K)) { return false; } @@ -7290,8 +7294,8 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session * return false; } - // Q4_0 has no raw fallback. Mark only accepted tensors for repacking. - if (src0->type == GGML_TYPE_Q4_0 && !src0->buffer) { + // Tiled quantized weights have no raw fallback. Mark only accepted tensors for repacking. + if ((src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q4_K || src0->type == GGML_TYPE_Q6_K) && !src0->buffer) { sess->needs_repack.insert(src0); } diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.c b/ggml/src/ggml-hexagon/htp/get-rows-ops.c index f354b813a..90f38fa56 100644 --- a/ggml/src/ggml-hexagon/htp/get-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.c @@ -217,7 +217,7 @@ GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int32_t, { compute_get_rows_q8_0((float GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int64_t, { compute_get_rows_q8_0((float *)dst_spad, src_spad, cur_elems); }) -static __attribute__((noinline)) void compute_get_rows_tiled(float * dst, const uint8_t * tile, uint32_t row, bool q4) { +static __attribute__((noinline)) void compute_get_rows_tiled(float * dst, const uint8_t * tile, uint32_t row, bool q4, bool q4_k) { const HVX_VectorPred first2 = Q6_Q_vsetq_R(2); const HVX_VectorPred first4 = Q6_Q_vsetq_R(4); HVX_Vector vq = Q6_V_vzero(); @@ -235,7 +235,9 @@ static __attribute__((noinline)) void compute_get_rows_tiled(float * dst, const const HVX_Vector lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F)); const HVX_Vector hi = Q6_Vub_vlsr_VubR(vq, 4); vq = Q6_V_lo_W(Q6_W_vshuff_VVR(hi, lo, -1)); - vq = Q6_Vb_vsub_VbVb(vq, Q6_Vb_vsplat_R(8)); + if (!q4_k) { + vq = Q6_Vb_vsub_VbVb(vq, Q6_Vb_vsplat_R(8)); + } } else { for (int group = 7; group >= 0; --group) { const HVX_Vector v = Q6_V_vror_VR(hvx_vmem(tile + group * VLEN), 2 * row); @@ -245,14 +247,46 @@ static __attribute__((noinline)) void compute_get_rows_tiled(float * dst, const } } const HVX_Vector scales = hvx_vmem(tile + (q4 ? 512 : 1024)); - const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row)); + const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, (q4_k ? 4 : 2) * row)); const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf)); const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq); const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16)); - const HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale); + HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale); + if (q4_k) { + const HVX_Vector offset_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 4 * row + 2)); + const HVX_Vector offset = Q6_V_lo_W(hvx_vec_f16_to_f32(offset_hf)); + values = hvx_vec_add_f32_f32(values, offset); + } *(HVX_Vector *) dst = values; } +static __attribute__((noinline)) void compute_get_rows_q6_k(float * dst, const uint8_t * tile, uint32_t row) { + const HVX_VectorPred first4 = Q6_Q_vsetq_R(4); + const HVX_VectorPred first16 = Q6_Q_vsetq_R(16 * sizeof(float)); + const HVX_Vector mask_0f = Q6_Vb_vsplat_R(0x0F); + const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + HVX_Vector vq = Q6_V_vzero(); + + for (int group = 7; group >= 0; --group) { + const HVX_Vector lo_plane = Q6_V_vror_VR(hvx_vmem(tile + (group >> 1) * VLEN), 4 * row); + const HVX_Vector hi_plane = Q6_V_vror_VR(hvx_vmem(tile + 512 + (group >> 2) * VLEN), 4 * row); + const HVX_Vector lo = (group & 1) ? Q6_Vub_vlsr_VubR(lo_plane, 4) : Q6_V_vand_VV(lo_plane, mask_0f); + const HVX_Vector hi = Q6_Vub_vlsr_VubR(hi_plane, 2 * (group & 3)); + const HVX_Vector packed = Q6_V_vor_VV(lo, Q6_Vw_vasl_VwR(Q6_V_vand_VV(hi, mask_03), 4)); + vq = Q6_V_vmux_QVV(first4, packed, Q6_V_vror_VR(vq, VLEN - 4)); + } + + const HVX_Vector scales = hvx_vmem(tile + 768); + const HVX_Vector scale_lo_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row)); + const HVX_Vector scale_hi_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 64 + 2 * row)); + const HVX_Vector scale_lo = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_lo_hf)); + const HVX_Vector scale_hi = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hi_hf)); + const HVX_Vector scale = Q6_V_vmux_QVV(first16, scale_lo, scale_hi); + const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(Q6_Vb_vsub_VbVb(vq, Q6_Vb_vsplat_R(32))); + const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16)); + *(HVX_Vector *) dst = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale); +} + struct get_rows_tiled_task { dma_addr_t tile_src_base; dma_addr_t dst_data; @@ -315,7 +349,9 @@ static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * dat const uint32_t tile_size = grctx->tile_size; const uint32_t tile_stride = grctx->tile_stride; const uint32_t dst_bytes = ne00 * sizeof(float); - const bool is_q4 = (octx->src[0]->type == HTP_TYPE_Q4_0); + const bool is_q4 = octx->src[0]->type == HTP_TYPE_Q4_0 || octx->src[0]->type == HTP_TYPE_Q4_K; + const bool is_q4_k = octx->src[0]->type == HTP_TYPE_Q4_K; + const bool is_q6_k = octx->src[0]->type == HTP_TYPE_Q6_K; for (uint32_t step = 0, spad_idx = 0; step < ir1 - ir0 && spad_idx < 2; ++step, ++spad_idx) { const uint32_t i = ir0 + step; @@ -343,7 +379,11 @@ static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * dat for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) { const uint8_t * tile = src_spad + k_tile * tile_stride; float * dst_block = dst_spad + k_tile * HTP_MM_HMX_TILE_N_COLS; - compute_get_rows_tiled(dst_block, tile, task.row, is_q4); + if (is_q6_k) { + compute_get_rows_q6_k(dst_block, tile, task.row); + } else { + compute_get_rows_tiled(dst_block, tile, task.row, is_q4, is_q4_k); + } } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); @@ -369,10 +409,12 @@ int op_get_rows(struct htp_ops_context * octx) { const struct htp_get_rows_kernel_params * kparams = (const struct htp_get_rows_kernel_params *) octx->kernel_params; if (octx->src[0]->type != HTP_TYPE_F32 && - octx->src[0]->type != HTP_TYPE_F16 && - octx->src[0]->type != HTP_TYPE_Q4_0 && - octx->src[0]->type != HTP_TYPE_Q8_0 && - octx->src[0]->type != HTP_TYPE_I32) { + octx->src[0]->type != HTP_TYPE_F16 && + octx->src[0]->type != HTP_TYPE_Q4_0 && + octx->src[0]->type != HTP_TYPE_Q4_K && + octx->src[0]->type != HTP_TYPE_Q6_K && + octx->src[0]->type != HTP_TYPE_Q8_0 && + octx->src[0]->type != HTP_TYPE_I32) { return HTP_STATUS_NO_SUPPORT; } @@ -426,7 +468,7 @@ int op_get_rows(struct htp_ops_context * octx) { grctx.task_start = task_start; grctx.tasks = tasks; grctx.tasks_per_thread = octx->ctx->mdev.count == 1 ? kparams->tasks_per_thread : fastdiv(tasks + n_threads - 1, &octx->n_threads_div); - grctx.tile_size = octx->src[0]->type == HTP_TYPE_Q4_0 ? HTP_MM_WEIGHT_TILE_SIZE_Q4_0 : HTP_MM_WEIGHT_TILE_SIZE_Q8_0; + grctx.tile_size = htp_mm_get_weight_tile_size(octx->src[0]->type); grctx.tile_stride = (grctx.tile_size + 127) & ~127; grctx.index_i32 = octx->src[1]->type == HTP_TYPE_I32; diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.h b/ggml/src/ggml-hexagon/htp/get-rows-ops.h index 06ca1ea74..69fa54464 100644 --- a/ggml/src/ggml-hexagon/htp/get-rows-ops.h +++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.h @@ -55,7 +55,7 @@ static inline void htp_get_rows_vtcm_layout_build( } if (kernel_type == HTP_GET_ROWS_KERNEL_TILED) { - const size_t tile_size = type == HTP_TYPE_Q4_0 ? HTP_MM_WEIGHT_TILE_SIZE_Q4_0 : HTP_MM_WEIGHT_TILE_SIZE_Q8_0; + const size_t tile_size = htp_mm_get_weight_tile_size(type); const size_t tile_stride = (tile_size + 127) & ~127; const uint32_t n_k_tiles = ne00 / HTP_MM_HMX_TILE_N_COLS; const size_t row_tiles_size = n_k_tiles > 0 ? (n_k_tiles * tile_stride) : tile_stride;