From 0ae0e1976670fac25f4e0ed864df30021accc764 Mon Sep 17 00:00:00 2001 From: kurquhar Date: Fri, 23 Oct 2026 14:58:44 -0700 Subject: [PATCH] hexagon: add SGN op (#30346) * hexagon : add SGN support Assisted-by: OpenCode * hexagon : optimize SGN with sign bit Assisted-by: OpenCode * hex-sgn: add f16 * hex-unary: update SGN to use generic unary_colapse and relax colapse checks to apply to more conditions * hex-unary: improve dims colapsing and run it in the host-side solver --------- Co-authored-by: Max Krasnyansky --- ggml/src/ggml-hexagon/htp-kparams.cpp | 55 ++++++++++++++++++++++++++- ggml/src/ggml-hexagon/htp/htp-ops.h | 1 + ggml/src/ggml-hexagon/htp/hvx-arith.h | 50 ++++++++++++++++++++++++ ggml/src/ggml-hexagon/htp/hvx-base.h | 11 ++++++ ggml/src/ggml-hexagon/htp/main.c | 1 + ggml/src/ggml-hexagon/htp/unary-ops.c | 47 +++++++++++++++++++++-- ggml/src/ggml-hexagon/htp/unary-ops.h | 31 +++------------ 7 files changed, 167 insertions(+), 29 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp-kparams.cpp b/ggml/src/ggml-hexagon/htp-kparams.cpp index 71040945d..589a8accf 100644 --- a/ggml/src/ggml-hexagon/htp-kparams.cpp +++ b/ggml/src/ggml-hexagon/htp-kparams.cpp @@ -73,6 +73,7 @@ htp_op_code htp_op_remap(const struct ggml_tensor * t) { case GGML_UNARY_OP_RELU: return HTP_OP_UNARY_RELU; case GGML_UNARY_OP_STEP: return HTP_OP_UNARY_STEP; case GGML_UNARY_OP_CEIL: return HTP_OP_UNARY_CEIL; + case GGML_UNARY_OP_SGN: return HTP_OP_UNARY_SGN; default: break; } break; @@ -1392,6 +1393,52 @@ static bool htp_kparams_solve_binary( return true; } +// collapse contiguous dims of elementwise ops so that rows fill at least one HVX vector +static void htp_unary_collapse_dims(struct ggml_tensor * src, struct ggml_tensor * dst, size_t elem_size) { + if (src->nb[0] != elem_size || dst->nb[0] != elem_size) { + return; + } + + while (src->ne[1] > 1 || src->ne[2] > 1 || src->ne[3] > 1) { + const size_t row_size = (size_t) src->ne[0] * elem_size; + const bool rows_contig = src->nb[1] == row_size && dst->nb[1] == (size_t) dst->ne[0] * elem_size; + const bool can_merge = (src->ne[1] == 1 && dst->ne[1] == 1) || + (rows_contig && (src->ne[0] == 1 || (row_size < 128 && row_size * src->ne[1] <= 2048))); + if (!can_merge) { + break; + } + src->ne[0] *= src->ne[1]; + src->ne[1] = src->ne[2]; + src->ne[2] = src->ne[3]; + src->ne[3] = 1; + dst->ne[0] *= dst->ne[1]; + dst->ne[1] = dst->ne[2]; + dst->ne[2] = dst->ne[3]; + dst->ne[3] = 1; + src->nb[1] = src->nb[2]; + src->nb[2] = src->nb[3]; + dst->nb[1] = dst->nb[2]; + dst->nb[2] = dst->nb[3]; + } + + // ne1 too big to merge in full: merge it in chunks of k rows (k divides ne1) + const size_t row_size = (size_t) src->ne[0] * elem_size; + if (src->ne[1] > 1 && row_size < 128 && src->nb[1] == row_size && dst->nb[1] == row_size) { + int64_t k = 2048 / row_size; + while (k > 1 && src->ne[1] % k != 0) { + k--; + } + if (k > 1) { + src->ne[0] *= k; + src->ne[1] /= k; + src->nb[1] *= k; + dst->ne[0] *= k; + dst->ne[1] /= k; + dst->nb[1] *= k; + } + } +} + static bool htp_kparams_solve_unary_impl( const struct ggml_hexagon_session * sess, uint32_t op, @@ -1435,6 +1482,7 @@ static bool htp_kparams_solve_unary_impl( case HTP_OP_UNARY_ABS: case HTP_OP_UNARY_STEP: case HTP_OP_UNARY_CEIL: + case HTP_OP_UNARY_SGN: break; default: return false; @@ -1451,9 +1499,14 @@ static bool htp_kparams_solve_unary_impl( struct ggml_tensor src0_c = *src0; struct ggml_tensor dst_c = *dst; if (htp_unary_is_elementwise(op)) { - HTP_UNARY_COLLAPSE_DIMS(src0_c.ne, src0_c.nb, dst_c.ne, dst_c.nb, elem_size); + htp_unary_collapse_dims(&src0_c, &dst_c, elem_size); src0 = &src0_c; dst = &dst_c; + for (int i = 0; i < HTP_OP_MAX_DIMS; i++) { + kparams->ne[i] = (uint32_t) src0->ne[i]; + kparams->src0_nb[i] = (uint32_t) src0->nb[i]; + kparams->dst_nb[i] = (uint32_t) dst->nb[i]; + } } const size_t dst_data_row_size = dst->ne[0] * ggml_type_size(dst->type); diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h index 50addf414..f3b2d6b73 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ops.h +++ b/ggml/src/ggml-hexagon/htp/htp-ops.h @@ -113,6 +113,7 @@ enum htp_op_code { HTP_OP_GLU_REGLU, HTP_OP_GLU_SIGMOID, HTP_OP_UNARY_CEIL, + HTP_OP_UNARY_SGN, HTP_OP_INVALID }; diff --git a/ggml/src/ggml-hexagon/htp/hvx-arith.h b/ggml/src/ggml-hexagon/htp/hvx-arith.h index 60da71fef..ec0de8e19 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-arith.h +++ b/ggml/src/ggml-hexagon/htp/hvx-arith.h @@ -677,6 +677,56 @@ static inline void hvx_step_f32_aa(uint8_t * restrict dst, const uint8_t * restr } } +static inline void hvx_sgn_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + HVX_Vector * restrict vsrc = (HVX_Vector *) src; + + const uint32_t elem_size = sizeof(float); + const uint32_t epv = 128 / elem_size; + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + vdst[i] = hvx_vec_sgn_f32(vsrc[i]); + } + if (nloe) { + HVX_Vector v = hvx_vec_sgn_f32(vsrc[i]); + hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v); + } +} + +static inline void hvx_sgn_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + HVX_Vector * restrict vsrc = (HVX_Vector *) src; + + const uint32_t elem_size = sizeof(_Float16); + const uint32_t epv = 128 / elem_size; + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); + vdst[i] = hvx_vec_f32_to_f16(hvx_vec_sgn_f32(Q6_V_lo_W(p)), hvx_vec_sgn_f32(Q6_V_hi_W(p))); + } + if (nloe) { + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); + HVX_Vector v = hvx_vec_f32_to_f16(hvx_vec_sgn_f32(Q6_V_lo_W(p)), hvx_vec_sgn_f32(Q6_V_hi_W(p))); + hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v); + } +} + static inline void hvx_step_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { assert((unsigned long) dst % 128 == 0); assert((unsigned long) src % 128 == 0); diff --git a/ggml/src/ggml-hexagon/htp/hvx-base.h b/ggml/src/ggml-hexagon/htp/hvx-base.h index 20e0d6e66..e87a30d14 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-base.h +++ b/ggml/src/ggml-hexagon/htp/hvx-base.h @@ -118,6 +118,17 @@ static inline HVX_Vector hvx_vec_step_f32(HVX_Vector v) { return Q6_V_vmux_QVV(q, one, zero); } +static inline HVX_Vector hvx_vec_sgn_f32(HVX_Vector v) { + const HVX_Vector zero = Q6_V_vzero(); + const HVX_Vector one = hvx_vec_splat_f32(1.0f); + const HVX_Vector sign_mask = Q6_V_vsplat_R(0x80000000); + const HVX_Vector mag = hvx_vec_abs_f32(v); + const HVX_Vector signed_one = Q6_V_vor_VV(one, Q6_V_vand_VV(v, sign_mask)); + // The ordered magnitude comparison maps NaN and both signed zeros to zero. + const HVX_VectorPred nonzero = Q6_Q_vcmp_gt_VsfVsf(mag, zero); + return Q6_V_vmux_QVV(nonzero, signed_one, zero); +} + static inline HVX_Vector hvx_vec_step_f16(HVX_Vector v) { const HVX_Vector zero = Q6_V_vzero(); const HVX_Vector one = hvx_vec_splat_f16((_Float16) 1.0f); diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index b22c3c944..d1a4afcd6 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -946,6 +946,7 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_UNARY_RELU: case HTP_OP_UNARY_STEP: case HTP_OP_UNARY_CEIL: + case HTP_OP_UNARY_SGN: case HTP_OP_L2_NORM: case HTP_OP_HADAMARD: return op_unary(octx); diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.c index 3a69337df..194569ab5 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.c +++ b/ggml/src/ggml-hexagon/htp/unary-ops.c @@ -440,6 +440,20 @@ static void ceil_f16(const void * restrict src, } } +static void sgn_f16(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_sgn_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0); + } +} + static void l2_norm_f16(const void * restrict src, void * restrict dst, const uint32_t num_rows, @@ -976,6 +990,20 @@ static void ceil_f32(const void * restrict src, } } +static void sgn_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_sgn_f32_aa(dst_local, src_local, ne0); + } +} + static void log_f32(const void * restrict src, void * restrict dst, const uint32_t num_rows, @@ -1094,6 +1122,11 @@ static void tile_ceil_f32(void * restrict dst, const void * restrict src, uint32 hvx_ceil_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); } +static void tile_sgn_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_sgn_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); +} + static void tri_apply_tile_f32(const void * restrict src, void * restrict dst, uint32_t tile_elems, uint32_t col_start, uint32_t i01, uint32_t ne0, int32_t ttype) { @@ -1738,6 +1771,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { case HTP_OP_UNARY_RELU: op_type = "relu-f32"; break; case HTP_OP_UNARY_STEP: op_type = is_f16 ? "step-f16" : "step-f32"; break; case HTP_OP_UNARY_CEIL: op_type = is_f16 ? "ceil-f16" : "ceil-f32"; break; + case HTP_OP_UNARY_SGN: op_type = is_f16 ? "sgn-f16" : "sgn-f32"; break; case HTP_OP_L2_NORM: op_type = is_f16 ? "l2norm-f16" : "l2norm-f32"; break; case HTP_OP_TRI: op_type = "tri-f32"; break; case HTP_OP_HADAMARD: op_type = is_f16 ? "hadamard-f16" : "hadamard-f32"; break; @@ -1752,9 +1786,9 @@ static int execute_op_unary(struct htp_ops_context * octx) { return HTP_STATUS_INVAL_PARAMS; } - const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; const size_t src0_elem_size = is_f16 ? sizeof(_Float16) : sizeof(float); const size_t dst_elem_size = (dst->type == HTP_TYPE_F16) ? sizeof(_Float16) : sizeof(float); + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; const size_t src0_data_row_size = src0->ne[0] * src0_elem_size; const size_t dst_data_row_size = dst->ne[0] * dst_elem_size; @@ -1872,6 +1906,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { case HTP_OP_UNARY_RELU: compute_func = (void *) tile_relu_f32; break; case HTP_OP_UNARY_STEP: compute_func = (void *) tile_step_f32; break; case HTP_OP_UNARY_CEIL: compute_func = (void *) tile_ceil_f32; break; + case HTP_OP_UNARY_SGN: compute_func = (void *) tile_sgn_f32; break; case HTP_OP_TRI: task_func = unary_thread_tiled_tri_f32; compute_func = (void *) tri_apply_tile_f32; @@ -1892,6 +1927,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { case HTP_OP_UNARY_LOG: compute_func = (void *) log_f16; break; case HTP_OP_UNARY_STEP: compute_func = (void *) step_f16; break; case HTP_OP_UNARY_CEIL: compute_func = (void *) ceil_f16; break; + case HTP_OP_UNARY_SGN: compute_func = (void *) sgn_f16; break; case HTP_OP_HADAMARD: compute_func = (void *) hadamard_f16; break; default: break; } @@ -1923,6 +1959,7 @@ static int execute_op_unary(struct htp_ops_context * octx) { case HTP_OP_UNARY_RELU: compute_func = (void *) relu_f32; break; case HTP_OP_UNARY_STEP: compute_func = (void *) step_f32; break; case HTP_OP_UNARY_CEIL: compute_func = (void *) ceil_f32; break; + case HTP_OP_UNARY_SGN: compute_func = (void *) sgn_f32; break; case HTP_OP_L2_NORM: compute_func = (void *) l2_norm_f32; break; case HTP_OP_HADAMARD: compute_func = (void *) hadamard_f32; break; case HTP_OP_TRI: @@ -1952,10 +1989,14 @@ int op_unary(struct htp_ops_context * octx) { return execute_op_unary(octx); } + const struct htp_unary_kernel_params * kparams = (const struct htp_unary_kernel_params *) octx->kernel_params; + struct htp_tensor src0_c = *octx->src[0]; struct htp_tensor dst_c = *octx->dst; - const size_t elem_size = (src0_c.type == HTP_TYPE_F16) ? sizeof(_Float16) : sizeof(float); - HTP_UNARY_COLLAPSE_DIMS(src0_c.ne, src0_c.nb, dst_c.ne, dst_c.nb, elem_size); + memcpy(src0_c.ne, kparams->ne, sizeof(src0_c.ne)); + memcpy(dst_c.ne, kparams->ne, sizeof(dst_c.ne)); + memcpy(src0_c.nb, kparams->src0_nb, sizeof(src0_c.nb)); + memcpy(dst_c.nb, kparams->dst_nb, sizeof(dst_c.nb)); const struct htp_tensor * orig_src0 = octx->src[0]; const struct htp_tensor * orig_dst = octx->dst; diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.h b/ggml/src/ggml-hexagon/htp/unary-ops.h index 93b9e0e75..b824544ff 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.h +++ b/ggml/src/ggml-hexagon/htp/unary-ops.h @@ -24,6 +24,11 @@ struct htp_unary_kernel_params { struct fastdiv_values div_ne02; struct fastdiv_values div_ne012; struct fastdiv_values div_tpr; + + // collapsed shape of elementwise ops + uint32_t ne[HTP_OP_MAX_DIMS]; + uint32_t src0_nb[HTP_OP_MAX_DIMS]; + uint32_t dst_nb[HTP_OP_MAX_DIMS]; }; #if defined(__cplusplus) @@ -56,6 +61,7 @@ static inline bool htp_op_is_unary(uint32_t opcode) { case HTP_OP_UNARY_RELU: case HTP_OP_UNARY_STEP: case HTP_OP_UNARY_CEIL: + case HTP_OP_UNARY_SGN: case HTP_OP_L2_NORM: case HTP_OP_TRI: case HTP_OP_HADAMARD: @@ -79,31 +85,6 @@ static inline bool htp_unary_is_elementwise(uint32_t opcode) { } } -#define HTP_UNARY_COLLAPSE_DIMS(src_ne, src_nb, dst_ne, dst_nb, elem_size) \ - do { \ - while (((src_ne)[1] > 1 || (src_ne)[2] > 1 || (src_ne)[3] > 1) && \ - (size_t) (src_nb)[0] == (elem_size) && \ - (size_t) (dst_nb)[0] == (elem_size) && \ - (((src_ne)[1] == 1 && (dst_ne)[1] == 1) || \ - ((size_t) (src_nb)[1] == (size_t) (src_ne)[0] * (size_t) (src_nb)[0] && \ - (size_t) (dst_nb)[1] == (size_t) (dst_ne)[0] * (size_t) (dst_nb)[0] && \ - ((src_ne)[0] == 1 || \ - (uint64_t) (src_ne)[0] * (uint64_t) (src_ne)[1] * (elem_size) <= 128)))) { \ - (src_ne)[0] *= (src_ne)[1]; \ - (src_ne)[1] = (src_ne)[2]; \ - (src_ne)[2] = (src_ne)[3]; \ - (src_ne)[3] = 1; \ - (dst_ne)[0] *= (dst_ne)[1]; \ - (dst_ne)[1] = (dst_ne)[2]; \ - (dst_ne)[2] = (dst_ne)[3]; \ - (dst_ne)[3] = 1; \ - (src_nb)[1] = (src_nb)[2]; \ - (src_nb)[2] = (src_nb)[3]; \ - (dst_nb)[1] = (dst_nb)[2]; \ - (dst_nb)[2] = (dst_nb)[3]; \ - } \ - } while (0) - struct htp_unary_vtcm_layout { size_t total_bytes; size_t off_src0;