hexagon: add SGN op (#30346)

* hexagon : add SGN support

Assisted-by: OpenCode

* hexagon : optimize SGN with sign bit

Assisted-by: OpenCode

* hex-sgn: add f16

* hex-unary: update SGN to use generic unary_colapse and relax colapse checks to apply to more conditions

* hex-unary: improve dims colapsing and run it in the host-side solver

---------

Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
This commit is contained in:
kurquharandMax Krasnyansky authored and GitHub committed 2026-10-23 14:58:44 -07:00
1 parent 21ec307146
commit 0ae0e19766
7 files changed
+167 -29

No files matched your search

+54 -1
View File
@@ -73,6 +73,7 @@ htp_op_code htp_op_remap(const struct ggml_tensor * t) {
case GGML_UNARY_OP_RELU: return HTP_OP_UNARY_RELU;
case GGML_UNARY_OP_STEP: return HTP_OP_UNARY_STEP;
case GGML_UNARY_OP_CEIL: return HTP_OP_UNARY_CEIL;
case GGML_UNARY_OP_SGN: return HTP_OP_UNARY_SGN;
default: break;
}
break;
@@ -1392,6 +1393,52 @@ static bool htp_kparams_solve_binary(
return true;
}
// collapse contiguous dims of elementwise ops so that rows fill at least one HVX vector
static void htp_unary_collapse_dims(struct ggml_tensor * src, struct ggml_tensor * dst, size_t elem_size) {
if (src->nb[0] != elem_size || dst->nb[0] != elem_size) {
return;
}
while (src->ne[1] > 1 || src->ne[2] > 1 || src->ne[3] > 1) {
const size_t row_size = (size_t) src->ne[0] * elem_size;
const bool rows_contig = src->nb[1] == row_size && dst->nb[1] == (size_t) dst->ne[0] * elem_size;
const bool can_merge = (src->ne[1] == 1 && dst->ne[1] == 1) ||
(rows_contig && (src->ne[0] == 1 || (row_size < 128 && row_size * src->ne[1] <= 2048)));
if (!can_merge) {
break;
}
src->ne[0] *= src->ne[1];
src->ne[1] = src->ne[2];
src->ne[2] = src->ne[3];
src->ne[3] = 1;
dst->ne[0] *= dst->ne[1];
dst->ne[1] = dst->ne[2];
dst->ne[2] = dst->ne[3];
dst->ne[3] = 1;
src->nb[1] = src->nb[2];
src->nb[2] = src->nb[3];
dst->nb[1] = dst->nb[2];
dst->nb[2] = dst->nb[3];
}
// ne1 too big to merge in full: merge it in chunks of k rows (k divides ne1)
const size_t row_size = (size_t) src->ne[0] * elem_size;
if (src->ne[1] > 1 && row_size < 128 && src->nb[1] == row_size && dst->nb[1] == row_size) {
int64_t k = 2048 / row_size;
while (k > 1 && src->ne[1] % k != 0) {
k--;
}
if (k > 1) {
src->ne[0] *= k;
src->ne[1] /= k;
src->nb[1] *= k;
dst->ne[0] *= k;
dst->ne[1] /= k;
dst->nb[1] *= k;
}
}
}
static bool htp_kparams_solve_unary_impl(
const struct ggml_hexagon_session * sess,
uint32_t op,
@@ -1435,6 +1482,7 @@ static bool htp_kparams_solve_unary_impl(
case HTP_OP_UNARY_ABS:
case HTP_OP_UNARY_STEP:
case HTP_OP_UNARY_CEIL:
case HTP_OP_UNARY_SGN:
break;
default:
return false;
@@ -1451,9 +1499,14 @@ static bool htp_kparams_solve_unary_impl(
struct ggml_tensor src0_c = *src0;
struct ggml_tensor dst_c = *dst;
if (htp_unary_is_elementwise(op)) {
HTP_UNARY_COLLAPSE_DIMS(src0_c.ne, src0_c.nb, dst_c.ne, dst_c.nb, elem_size);
htp_unary_collapse_dims(&src0_c, &dst_c, elem_size);
src0 = &src0_c;
dst = &dst_c;
for (int i = 0; i < HTP_OP_MAX_DIMS; i++) {
kparams->ne[i] = (uint32_t) src0->ne[i];
kparams->src0_nb[i] = (uint32_t) src0->nb[i];
kparams->dst_nb[i] = (uint32_t) dst->nb[i];
}
}
const size_t dst_data_row_size = dst->ne[0] * ggml_type_size(dst->type);
+1
View File
@@ -113,6 +113,7 @@ enum htp_op_code {
HTP_OP_GLU_REGLU,
HTP_OP_GLU_SIGMOID,
HTP_OP_UNARY_CEIL,
HTP_OP_UNARY_SGN,
HTP_OP_INVALID
};
+50
View File
@@ -677,6 +677,56 @@ static inline void hvx_step_f32_aa(uint8_t * restrict dst, const uint8_t * restr
}
}
static inline void hvx_sgn_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
HVX_Vector * restrict vdst = (HVX_Vector *) dst;
HVX_Vector * restrict vsrc = (HVX_Vector *) src;
const uint32_t elem_size = sizeof(float);
const uint32_t epv = 128 / elem_size;
const uint32_t nvec = n / epv;
const uint32_t nloe = n % epv;
uint32_t i = 0;
_Pragma("unroll(4)")
for (; i < nvec; i++) {
vdst[i] = hvx_vec_sgn_f32(vsrc[i]);
}
if (nloe) {
HVX_Vector v = hvx_vec_sgn_f32(vsrc[i]);
hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v);
}
}
static inline void hvx_sgn_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
HVX_Vector * restrict vdst = (HVX_Vector *) dst;
HVX_Vector * restrict vsrc = (HVX_Vector *) src;
const uint32_t elem_size = sizeof(_Float16);
const uint32_t epv = 128 / elem_size;
const uint32_t nvec = n / epv;
const uint32_t nloe = n % epv;
uint32_t i = 0;
_Pragma("unroll(4)")
for (; i < nvec; i++) {
HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]);
vdst[i] = hvx_vec_f32_to_f16(hvx_vec_sgn_f32(Q6_V_lo_W(p)), hvx_vec_sgn_f32(Q6_V_hi_W(p)));
}
if (nloe) {
HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]);
HVX_Vector v = hvx_vec_f32_to_f16(hvx_vec_sgn_f32(Q6_V_lo_W(p)), hvx_vec_sgn_f32(Q6_V_hi_W(p)));
hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v);
}
}
static inline void hvx_step_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
+11
View File
@@ -118,6 +118,17 @@ static inline HVX_Vector hvx_vec_step_f32(HVX_Vector v) {
return Q6_V_vmux_QVV(q, one, zero);
}
static inline HVX_Vector hvx_vec_sgn_f32(HVX_Vector v) {
const HVX_Vector zero = Q6_V_vzero();
const HVX_Vector one = hvx_vec_splat_f32(1.0f);
const HVX_Vector sign_mask = Q6_V_vsplat_R(0x80000000);
const HVX_Vector mag = hvx_vec_abs_f32(v);
const HVX_Vector signed_one = Q6_V_vor_VV(one, Q6_V_vand_VV(v, sign_mask));
// The ordered magnitude comparison maps NaN and both signed zeros to zero.
const HVX_VectorPred nonzero = Q6_Q_vcmp_gt_VsfVsf(mag, zero);
return Q6_V_vmux_QVV(nonzero, signed_one, zero);
}
static inline HVX_Vector hvx_vec_step_f16(HVX_Vector v) {
const HVX_Vector zero = Q6_V_vzero();
const HVX_Vector one = hvx_vec_splat_f16((_Float16) 1.0f);
+1
View File
@@ -946,6 +946,7 @@ static int execute_op(struct htp_ops_context * octx) {
case HTP_OP_UNARY_RELU:
case HTP_OP_UNARY_STEP:
case HTP_OP_UNARY_CEIL:
case HTP_OP_UNARY_SGN:
case HTP_OP_L2_NORM:
case HTP_OP_HADAMARD:
return op_unary(octx);
+44 -3
View File
@@ -440,6 +440,20 @@ static void ceil_f16(const void * restrict src,
}
}
static void sgn_f16(const void * restrict src,
void * restrict dst,
const uint32_t num_rows,
const struct htp_unary_context * uctx) {
htp_unary_op_preamble;
for (uint32_t ir = 0; ir < num_rows; ir++) {
const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
hvx_sgn_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0);
}
}
static void l2_norm_f16(const void * restrict src,
void * restrict dst,
const uint32_t num_rows,
@@ -976,6 +990,20 @@ static void ceil_f32(const void * restrict src,
}
}
static void sgn_f32(const void * restrict src,
void * restrict dst,
const uint32_t num_rows,
const struct htp_unary_context * uctx) {
htp_unary_op_preamble;
for (uint32_t ir = 0; ir < num_rows; ir++) {
const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
hvx_sgn_f32_aa(dst_local, src_local, ne0);
}
}
static void log_f32(const void * restrict src,
void * restrict dst,
const uint32_t num_rows,
@@ -1094,6 +1122,11 @@ static void tile_ceil_f32(void * restrict dst, const void * restrict src, uint32
hvx_ceil_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw);
}
static void tile_sgn_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) {
(void) uctx;
hvx_sgn_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw);
}
static void tri_apply_tile_f32(const void * restrict src, void * restrict dst,
uint32_t tile_elems, uint32_t col_start, uint32_t i01,
uint32_t ne0, int32_t ttype) {
@@ -1738,6 +1771,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {
case HTP_OP_UNARY_RELU: op_type = "relu-f32"; break;
case HTP_OP_UNARY_STEP: op_type = is_f16 ? "step-f16" : "step-f32"; break;
case HTP_OP_UNARY_CEIL: op_type = is_f16 ? "ceil-f16" : "ceil-f32"; break;
case HTP_OP_UNARY_SGN: op_type = is_f16 ? "sgn-f16" : "sgn-f32"; break;
case HTP_OP_L2_NORM: op_type = is_f16 ? "l2norm-f16" : "l2norm-f32"; break;
case HTP_OP_TRI: op_type = "tri-f32"; break;
case HTP_OP_HADAMARD: op_type = is_f16 ? "hadamard-f16" : "hadamard-f32"; break;
@@ -1752,9 +1786,9 @@ static int execute_op_unary(struct htp_ops_context * octx) {
return HTP_STATUS_INVAL_PARAMS;
}
const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
const size_t src0_elem_size = is_f16 ? sizeof(_Float16) : sizeof(float);
const size_t dst_elem_size = (dst->type == HTP_TYPE_F16) ? sizeof(_Float16) : sizeof(float);
const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
const size_t src0_data_row_size = src0->ne[0] * src0_elem_size;
const size_t dst_data_row_size = dst->ne[0] * dst_elem_size;
@@ -1872,6 +1906,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {
case HTP_OP_UNARY_RELU: compute_func = (void *) tile_relu_f32; break;
case HTP_OP_UNARY_STEP: compute_func = (void *) tile_step_f32; break;
case HTP_OP_UNARY_CEIL: compute_func = (void *) tile_ceil_f32; break;
case HTP_OP_UNARY_SGN: compute_func = (void *) tile_sgn_f32; break;
case HTP_OP_TRI:
task_func = unary_thread_tiled_tri_f32;
compute_func = (void *) tri_apply_tile_f32;
@@ -1892,6 +1927,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {
case HTP_OP_UNARY_LOG: compute_func = (void *) log_f16; break;
case HTP_OP_UNARY_STEP: compute_func = (void *) step_f16; break;
case HTP_OP_UNARY_CEIL: compute_func = (void *) ceil_f16; break;
case HTP_OP_UNARY_SGN: compute_func = (void *) sgn_f16; break;
case HTP_OP_HADAMARD: compute_func = (void *) hadamard_f16; break;
default: break;
}
@@ -1923,6 +1959,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {
case HTP_OP_UNARY_RELU: compute_func = (void *) relu_f32; break;
case HTP_OP_UNARY_STEP: compute_func = (void *) step_f32; break;
case HTP_OP_UNARY_CEIL: compute_func = (void *) ceil_f32; break;
case HTP_OP_UNARY_SGN: compute_func = (void *) sgn_f32; break;
case HTP_OP_L2_NORM: compute_func = (void *) l2_norm_f32; break;
case HTP_OP_HADAMARD: compute_func = (void *) hadamard_f32; break;
case HTP_OP_TRI:
@@ -1952,10 +1989,14 @@ int op_unary(struct htp_ops_context * octx) {
return execute_op_unary(octx);
}
const struct htp_unary_kernel_params * kparams = (const struct htp_unary_kernel_params *) octx->kernel_params;
struct htp_tensor src0_c = *octx->src[0];
struct htp_tensor dst_c = *octx->dst;
const size_t elem_size = (src0_c.type == HTP_TYPE_F16) ? sizeof(_Float16) : sizeof(float);
HTP_UNARY_COLLAPSE_DIMS(src0_c.ne, src0_c.nb, dst_c.ne, dst_c.nb, elem_size);
memcpy(src0_c.ne, kparams->ne, sizeof(src0_c.ne));
memcpy(dst_c.ne, kparams->ne, sizeof(dst_c.ne));
memcpy(src0_c.nb, kparams->src0_nb, sizeof(src0_c.nb));
memcpy(dst_c.nb, kparams->dst_nb, sizeof(dst_c.nb));
const struct htp_tensor * orig_src0 = octx->src[0];
const struct htp_tensor * orig_dst = octx->dst;
+6 -25
View File
@@ -24,6 +24,11 @@ struct htp_unary_kernel_params {
struct fastdiv_values div_ne02;
struct fastdiv_values div_ne012;
struct fastdiv_values div_tpr;
// collapsed shape of elementwise ops
uint32_t ne[HTP_OP_MAX_DIMS];
uint32_t src0_nb[HTP_OP_MAX_DIMS];
uint32_t dst_nb[HTP_OP_MAX_DIMS];
};
#if defined(__cplusplus)
@@ -56,6 +61,7 @@ static inline bool htp_op_is_unary(uint32_t opcode) {
case HTP_OP_UNARY_RELU:
case HTP_OP_UNARY_STEP:
case HTP_OP_UNARY_CEIL:
case HTP_OP_UNARY_SGN:
case HTP_OP_L2_NORM:
case HTP_OP_TRI:
case HTP_OP_HADAMARD:
@@ -79,31 +85,6 @@ static inline bool htp_unary_is_elementwise(uint32_t opcode) {
}
}
#define HTP_UNARY_COLLAPSE_DIMS(src_ne, src_nb, dst_ne, dst_nb, elem_size) \
do { \
while (((src_ne)[1] > 1 || (src_ne)[2] > 1 || (src_ne)[3] > 1) && \
(size_t) (src_nb)[0] == (elem_size) && \
(size_t) (dst_nb)[0] == (elem_size) && \
(((src_ne)[1] == 1 && (dst_ne)[1] == 1) || \
((size_t) (src_nb)[1] == (size_t) (src_ne)[0] * (size_t) (src_nb)[0] && \
(size_t) (dst_nb)[1] == (size_t) (dst_ne)[0] * (size_t) (dst_nb)[0] && \
((src_ne)[0] == 1 || \
(uint64_t) (src_ne)[0] * (uint64_t) (src_ne)[1] * (elem_size) <= 128)))) { \
(src_ne)[0] *= (src_ne)[1]; \
(src_ne)[1] = (src_ne)[2]; \
(src_ne)[2] = (src_ne)[3]; \
(src_ne)[3] = 1; \
(dst_ne)[0] *= (dst_ne)[1]; \
(dst_ne)[1] = (dst_ne)[2]; \
(dst_ne)[2] = (dst_ne)[3]; \
(dst_ne)[3] = 1; \
(src_nb)[1] = (src_nb)[2]; \
(src_nb)[2] = (src_nb)[3]; \
(dst_nb)[1] = (dst_nb)[2]; \
(dst_nb)[2] = (dst_nb)[3]; \
} \
} while (0)
struct htp_unary_vtcm_layout {
size_t total_bytes;
size_t off_src0;