hexagon: add SGN op (#30346)
* hexagon : add SGN support Assisted-by: OpenCode * hexagon : optimize SGN with sign bit Assisted-by: OpenCode * hex-sgn: add f16 * hex-unary: update SGN to use generic unary_colapse and relax colapse checks to apply to more conditions * hex-unary: improve dims colapsing and run it in the host-side solver --------- Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
This commit is contained in:
1 parent
21ec307146
commit
0ae0e19766
7 files changed
+167
-29
No files matched your search
@@ -73,6 +73,7 @@ htp_op_code htp_op_remap(const struct ggml_tensor * t) {
|
||||
case GGML_UNARY_OP_RELU: return HTP_OP_UNARY_RELU;
|
||||
case GGML_UNARY_OP_STEP: return HTP_OP_UNARY_STEP;
|
||||
case GGML_UNARY_OP_CEIL: return HTP_OP_UNARY_CEIL;
|
||||
case GGML_UNARY_OP_SGN: return HTP_OP_UNARY_SGN;
|
||||
default: break;
|
||||
}
|
||||
break;
|
||||
@@ -1392,6 +1393,52 @@ static bool htp_kparams_solve_binary(
|
||||
return true;
|
||||
}
|
||||
|
||||
// collapse contiguous dims of elementwise ops so that rows fill at least one HVX vector
|
||||
static void htp_unary_collapse_dims(struct ggml_tensor * src, struct ggml_tensor * dst, size_t elem_size) {
|
||||
if (src->nb[0] != elem_size || dst->nb[0] != elem_size) {
|
||||
return;
|
||||
}
|
||||
|
||||
while (src->ne[1] > 1 || src->ne[2] > 1 || src->ne[3] > 1) {
|
||||
const size_t row_size = (size_t) src->ne[0] * elem_size;
|
||||
const bool rows_contig = src->nb[1] == row_size && dst->nb[1] == (size_t) dst->ne[0] * elem_size;
|
||||
const bool can_merge = (src->ne[1] == 1 && dst->ne[1] == 1) ||
|
||||
(rows_contig && (src->ne[0] == 1 || (row_size < 128 && row_size * src->ne[1] <= 2048)));
|
||||
if (!can_merge) {
|
||||
break;
|
||||
}
|
||||
src->ne[0] *= src->ne[1];
|
||||
src->ne[1] = src->ne[2];
|
||||
src->ne[2] = src->ne[3];
|
||||
src->ne[3] = 1;
|
||||
dst->ne[0] *= dst->ne[1];
|
||||
dst->ne[1] = dst->ne[2];
|
||||
dst->ne[2] = dst->ne[3];
|
||||
dst->ne[3] = 1;
|
||||
src->nb[1] = src->nb[2];
|
||||
src->nb[2] = src->nb[3];
|
||||
dst->nb[1] = dst->nb[2];
|
||||
dst->nb[2] = dst->nb[3];
|
||||
}
|
||||
|
||||
// ne1 too big to merge in full: merge it in chunks of k rows (k divides ne1)
|
||||
const size_t row_size = (size_t) src->ne[0] * elem_size;
|
||||
if (src->ne[1] > 1 && row_size < 128 && src->nb[1] == row_size && dst->nb[1] == row_size) {
|
||||
int64_t k = 2048 / row_size;
|
||||
while (k > 1 && src->ne[1] % k != 0) {
|
||||
k--;
|
||||
}
|
||||
if (k > 1) {
|
||||
src->ne[0] *= k;
|
||||
src->ne[1] /= k;
|
||||
src->nb[1] *= k;
|
||||
dst->ne[0] *= k;
|
||||
dst->ne[1] /= k;
|
||||
dst->nb[1] *= k;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static bool htp_kparams_solve_unary_impl(
|
||||
const struct ggml_hexagon_session * sess,
|
||||
uint32_t op,
|
||||
@@ -1435,6 +1482,7 @@ static bool htp_kparams_solve_unary_impl(
|
||||
case HTP_OP_UNARY_ABS:
|
||||
case HTP_OP_UNARY_STEP:
|
||||
case HTP_OP_UNARY_CEIL:
|
||||
case HTP_OP_UNARY_SGN:
|
||||
break;
|
||||
default:
|
||||
return false;
|
||||
@@ -1451,9 +1499,14 @@ static bool htp_kparams_solve_unary_impl(
|
||||
struct ggml_tensor src0_c = *src0;
|
||||
struct ggml_tensor dst_c = *dst;
|
||||
if (htp_unary_is_elementwise(op)) {
|
||||
HTP_UNARY_COLLAPSE_DIMS(src0_c.ne, src0_c.nb, dst_c.ne, dst_c.nb, elem_size);
|
||||
htp_unary_collapse_dims(&src0_c, &dst_c, elem_size);
|
||||
src0 = &src0_c;
|
||||
dst = &dst_c;
|
||||
for (int i = 0; i < HTP_OP_MAX_DIMS; i++) {
|
||||
kparams->ne[i] = (uint32_t) src0->ne[i];
|
||||
kparams->src0_nb[i] = (uint32_t) src0->nb[i];
|
||||
kparams->dst_nb[i] = (uint32_t) dst->nb[i];
|
||||
}
|
||||
}
|
||||
|
||||
const size_t dst_data_row_size = dst->ne[0] * ggml_type_size(dst->type);
|
||||
|
||||
@@ -113,6 +113,7 @@ enum htp_op_code {
|
||||
HTP_OP_GLU_REGLU,
|
||||
HTP_OP_GLU_SIGMOID,
|
||||
HTP_OP_UNARY_CEIL,
|
||||
HTP_OP_UNARY_SGN,
|
||||
|
||||
HTP_OP_INVALID
|
||||
};
|
||||
|
||||
@@ -677,6 +677,56 @@ static inline void hvx_step_f32_aa(uint8_t * restrict dst, const uint8_t * restr
|
||||
}
|
||||
}
|
||||
|
||||
static inline void hvx_sgn_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
|
||||
assert((unsigned long) dst % 128 == 0);
|
||||
assert((unsigned long) src % 128 == 0);
|
||||
|
||||
HVX_Vector * restrict vdst = (HVX_Vector *) dst;
|
||||
HVX_Vector * restrict vsrc = (HVX_Vector *) src;
|
||||
|
||||
const uint32_t elem_size = sizeof(float);
|
||||
const uint32_t epv = 128 / elem_size;
|
||||
const uint32_t nvec = n / epv;
|
||||
const uint32_t nloe = n % epv;
|
||||
|
||||
uint32_t i = 0;
|
||||
|
||||
_Pragma("unroll(4)")
|
||||
for (; i < nvec; i++) {
|
||||
vdst[i] = hvx_vec_sgn_f32(vsrc[i]);
|
||||
}
|
||||
if (nloe) {
|
||||
HVX_Vector v = hvx_vec_sgn_f32(vsrc[i]);
|
||||
hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void hvx_sgn_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
|
||||
assert((unsigned long) dst % 128 == 0);
|
||||
assert((unsigned long) src % 128 == 0);
|
||||
|
||||
HVX_Vector * restrict vdst = (HVX_Vector *) dst;
|
||||
HVX_Vector * restrict vsrc = (HVX_Vector *) src;
|
||||
|
||||
const uint32_t elem_size = sizeof(_Float16);
|
||||
const uint32_t epv = 128 / elem_size;
|
||||
const uint32_t nvec = n / epv;
|
||||
const uint32_t nloe = n % epv;
|
||||
|
||||
uint32_t i = 0;
|
||||
|
||||
_Pragma("unroll(4)")
|
||||
for (; i < nvec; i++) {
|
||||
HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]);
|
||||
vdst[i] = hvx_vec_f32_to_f16(hvx_vec_sgn_f32(Q6_V_lo_W(p)), hvx_vec_sgn_f32(Q6_V_hi_W(p)));
|
||||
}
|
||||
if (nloe) {
|
||||
HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]);
|
||||
HVX_Vector v = hvx_vec_f32_to_f16(hvx_vec_sgn_f32(Q6_V_lo_W(p)), hvx_vec_sgn_f32(Q6_V_hi_W(p)));
|
||||
hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void hvx_step_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
|
||||
assert((unsigned long) dst % 128 == 0);
|
||||
assert((unsigned long) src % 128 == 0);
|
||||
|
||||
@@ -118,6 +118,17 @@ static inline HVX_Vector hvx_vec_step_f32(HVX_Vector v) {
|
||||
return Q6_V_vmux_QVV(q, one, zero);
|
||||
}
|
||||
|
||||
static inline HVX_Vector hvx_vec_sgn_f32(HVX_Vector v) {
|
||||
const HVX_Vector zero = Q6_V_vzero();
|
||||
const HVX_Vector one = hvx_vec_splat_f32(1.0f);
|
||||
const HVX_Vector sign_mask = Q6_V_vsplat_R(0x80000000);
|
||||
const HVX_Vector mag = hvx_vec_abs_f32(v);
|
||||
const HVX_Vector signed_one = Q6_V_vor_VV(one, Q6_V_vand_VV(v, sign_mask));
|
||||
// The ordered magnitude comparison maps NaN and both signed zeros to zero.
|
||||
const HVX_VectorPred nonzero = Q6_Q_vcmp_gt_VsfVsf(mag, zero);
|
||||
return Q6_V_vmux_QVV(nonzero, signed_one, zero);
|
||||
}
|
||||
|
||||
static inline HVX_Vector hvx_vec_step_f16(HVX_Vector v) {
|
||||
const HVX_Vector zero = Q6_V_vzero();
|
||||
const HVX_Vector one = hvx_vec_splat_f16((_Float16) 1.0f);
|
||||
|
||||
@@ -946,6 +946,7 @@ static int execute_op(struct htp_ops_context * octx) {
|
||||
case HTP_OP_UNARY_RELU:
|
||||
case HTP_OP_UNARY_STEP:
|
||||
case HTP_OP_UNARY_CEIL:
|
||||
case HTP_OP_UNARY_SGN:
|
||||
case HTP_OP_L2_NORM:
|
||||
case HTP_OP_HADAMARD:
|
||||
return op_unary(octx);
|
||||
|
||||
@@ -440,6 +440,20 @@ static void ceil_f16(const void * restrict src,
|
||||
}
|
||||
}
|
||||
|
||||
static void sgn_f16(const void * restrict src,
|
||||
void * restrict dst,
|
||||
const uint32_t num_rows,
|
||||
const struct htp_unary_context * uctx) {
|
||||
htp_unary_op_preamble;
|
||||
|
||||
for (uint32_t ir = 0; ir < num_rows; ir++) {
|
||||
const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
|
||||
uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
|
||||
|
||||
hvx_sgn_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0);
|
||||
}
|
||||
}
|
||||
|
||||
static void l2_norm_f16(const void * restrict src,
|
||||
void * restrict dst,
|
||||
const uint32_t num_rows,
|
||||
@@ -976,6 +990,20 @@ static void ceil_f32(const void * restrict src,
|
||||
}
|
||||
}
|
||||
|
||||
static void sgn_f32(const void * restrict src,
|
||||
void * restrict dst,
|
||||
const uint32_t num_rows,
|
||||
const struct htp_unary_context * uctx) {
|
||||
htp_unary_op_preamble;
|
||||
|
||||
for (uint32_t ir = 0; ir < num_rows; ir++) {
|
||||
const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
|
||||
uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
|
||||
|
||||
hvx_sgn_f32_aa(dst_local, src_local, ne0);
|
||||
}
|
||||
}
|
||||
|
||||
static void log_f32(const void * restrict src,
|
||||
void * restrict dst,
|
||||
const uint32_t num_rows,
|
||||
@@ -1094,6 +1122,11 @@ static void tile_ceil_f32(void * restrict dst, const void * restrict src, uint32
|
||||
hvx_ceil_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw);
|
||||
}
|
||||
|
||||
static void tile_sgn_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) {
|
||||
(void) uctx;
|
||||
hvx_sgn_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw);
|
||||
}
|
||||
|
||||
static void tri_apply_tile_f32(const void * restrict src, void * restrict dst,
|
||||
uint32_t tile_elems, uint32_t col_start, uint32_t i01,
|
||||
uint32_t ne0, int32_t ttype) {
|
||||
@@ -1738,6 +1771,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {
|
||||
case HTP_OP_UNARY_RELU: op_type = "relu-f32"; break;
|
||||
case HTP_OP_UNARY_STEP: op_type = is_f16 ? "step-f16" : "step-f32"; break;
|
||||
case HTP_OP_UNARY_CEIL: op_type = is_f16 ? "ceil-f16" : "ceil-f32"; break;
|
||||
case HTP_OP_UNARY_SGN: op_type = is_f16 ? "sgn-f16" : "sgn-f32"; break;
|
||||
case HTP_OP_L2_NORM: op_type = is_f16 ? "l2norm-f16" : "l2norm-f32"; break;
|
||||
case HTP_OP_TRI: op_type = "tri-f32"; break;
|
||||
case HTP_OP_HADAMARD: op_type = is_f16 ? "hadamard-f16" : "hadamard-f32"; break;
|
||||
@@ -1752,9 +1786,9 @@ static int execute_op_unary(struct htp_ops_context * octx) {
|
||||
return HTP_STATUS_INVAL_PARAMS;
|
||||
}
|
||||
|
||||
const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
|
||||
const size_t src0_elem_size = is_f16 ? sizeof(_Float16) : sizeof(float);
|
||||
const size_t dst_elem_size = (dst->type == HTP_TYPE_F16) ? sizeof(_Float16) : sizeof(float);
|
||||
const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
|
||||
const size_t src0_data_row_size = src0->ne[0] * src0_elem_size;
|
||||
const size_t dst_data_row_size = dst->ne[0] * dst_elem_size;
|
||||
|
||||
@@ -1872,6 +1906,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {
|
||||
case HTP_OP_UNARY_RELU: compute_func = (void *) tile_relu_f32; break;
|
||||
case HTP_OP_UNARY_STEP: compute_func = (void *) tile_step_f32; break;
|
||||
case HTP_OP_UNARY_CEIL: compute_func = (void *) tile_ceil_f32; break;
|
||||
case HTP_OP_UNARY_SGN: compute_func = (void *) tile_sgn_f32; break;
|
||||
case HTP_OP_TRI:
|
||||
task_func = unary_thread_tiled_tri_f32;
|
||||
compute_func = (void *) tri_apply_tile_f32;
|
||||
@@ -1892,6 +1927,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {
|
||||
case HTP_OP_UNARY_LOG: compute_func = (void *) log_f16; break;
|
||||
case HTP_OP_UNARY_STEP: compute_func = (void *) step_f16; break;
|
||||
case HTP_OP_UNARY_CEIL: compute_func = (void *) ceil_f16; break;
|
||||
case HTP_OP_UNARY_SGN: compute_func = (void *) sgn_f16; break;
|
||||
case HTP_OP_HADAMARD: compute_func = (void *) hadamard_f16; break;
|
||||
default: break;
|
||||
}
|
||||
@@ -1923,6 +1959,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {
|
||||
case HTP_OP_UNARY_RELU: compute_func = (void *) relu_f32; break;
|
||||
case HTP_OP_UNARY_STEP: compute_func = (void *) step_f32; break;
|
||||
case HTP_OP_UNARY_CEIL: compute_func = (void *) ceil_f32; break;
|
||||
case HTP_OP_UNARY_SGN: compute_func = (void *) sgn_f32; break;
|
||||
case HTP_OP_L2_NORM: compute_func = (void *) l2_norm_f32; break;
|
||||
case HTP_OP_HADAMARD: compute_func = (void *) hadamard_f32; break;
|
||||
case HTP_OP_TRI:
|
||||
@@ -1952,10 +1989,14 @@ int op_unary(struct htp_ops_context * octx) {
|
||||
return execute_op_unary(octx);
|
||||
}
|
||||
|
||||
const struct htp_unary_kernel_params * kparams = (const struct htp_unary_kernel_params *) octx->kernel_params;
|
||||
|
||||
struct htp_tensor src0_c = *octx->src[0];
|
||||
struct htp_tensor dst_c = *octx->dst;
|
||||
const size_t elem_size = (src0_c.type == HTP_TYPE_F16) ? sizeof(_Float16) : sizeof(float);
|
||||
HTP_UNARY_COLLAPSE_DIMS(src0_c.ne, src0_c.nb, dst_c.ne, dst_c.nb, elem_size);
|
||||
memcpy(src0_c.ne, kparams->ne, sizeof(src0_c.ne));
|
||||
memcpy(dst_c.ne, kparams->ne, sizeof(dst_c.ne));
|
||||
memcpy(src0_c.nb, kparams->src0_nb, sizeof(src0_c.nb));
|
||||
memcpy(dst_c.nb, kparams->dst_nb, sizeof(dst_c.nb));
|
||||
|
||||
const struct htp_tensor * orig_src0 = octx->src[0];
|
||||
const struct htp_tensor * orig_dst = octx->dst;
|
||||
|
||||
@@ -24,6 +24,11 @@ struct htp_unary_kernel_params {
|
||||
struct fastdiv_values div_ne02;
|
||||
struct fastdiv_values div_ne012;
|
||||
struct fastdiv_values div_tpr;
|
||||
|
||||
// collapsed shape of elementwise ops
|
||||
uint32_t ne[HTP_OP_MAX_DIMS];
|
||||
uint32_t src0_nb[HTP_OP_MAX_DIMS];
|
||||
uint32_t dst_nb[HTP_OP_MAX_DIMS];
|
||||
};
|
||||
|
||||
#if defined(__cplusplus)
|
||||
@@ -56,6 +61,7 @@ static inline bool htp_op_is_unary(uint32_t opcode) {
|
||||
case HTP_OP_UNARY_RELU:
|
||||
case HTP_OP_UNARY_STEP:
|
||||
case HTP_OP_UNARY_CEIL:
|
||||
case HTP_OP_UNARY_SGN:
|
||||
case HTP_OP_L2_NORM:
|
||||
case HTP_OP_TRI:
|
||||
case HTP_OP_HADAMARD:
|
||||
@@ -79,31 +85,6 @@ static inline bool htp_unary_is_elementwise(uint32_t opcode) {
|
||||
}
|
||||
}
|
||||
|
||||
#define HTP_UNARY_COLLAPSE_DIMS(src_ne, src_nb, dst_ne, dst_nb, elem_size) \
|
||||
do { \
|
||||
while (((src_ne)[1] > 1 || (src_ne)[2] > 1 || (src_ne)[3] > 1) && \
|
||||
(size_t) (src_nb)[0] == (elem_size) && \
|
||||
(size_t) (dst_nb)[0] == (elem_size) && \
|
||||
(((src_ne)[1] == 1 && (dst_ne)[1] == 1) || \
|
||||
((size_t) (src_nb)[1] == (size_t) (src_ne)[0] * (size_t) (src_nb)[0] && \
|
||||
(size_t) (dst_nb)[1] == (size_t) (dst_ne)[0] * (size_t) (dst_nb)[0] && \
|
||||
((src_ne)[0] == 1 || \
|
||||
(uint64_t) (src_ne)[0] * (uint64_t) (src_ne)[1] * (elem_size) <= 128)))) { \
|
||||
(src_ne)[0] *= (src_ne)[1]; \
|
||||
(src_ne)[1] = (src_ne)[2]; \
|
||||
(src_ne)[2] = (src_ne)[3]; \
|
||||
(src_ne)[3] = 1; \
|
||||
(dst_ne)[0] *= (dst_ne)[1]; \
|
||||
(dst_ne)[1] = (dst_ne)[2]; \
|
||||
(dst_ne)[2] = (dst_ne)[3]; \
|
||||
(dst_ne)[3] = 1; \
|
||||
(src_nb)[1] = (src_nb)[2]; \
|
||||
(src_nb)[2] = (src_nb)[3]; \
|
||||
(dst_nb)[1] = (dst_nb)[2]; \
|
||||
(dst_nb)[2] = (dst_nb)[3]; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
struct htp_unary_vtcm_layout {
|
||||
size_t total_bytes;
|
||||
size_t off_src0;
|
||||
|
||||
Reference in new issue
Block a user