diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index 533f51a0e..a32ed5919 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -6593,6 +6593,37 @@ static bool ggml_hexagon_precompute_concat_params( } } + const uint32_t max_gather_ne0 = 128 / type_size; + const uint32_t total_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]); + const bool b_dense = (src1->nb[0] == type_size); + const bool gather_strides_ok = + src0->nb[1] >= (size_t) src0->ne[0] * type_size && + (b_dense ? src1->nb[1] >= (size_t) src1->ne[0] * type_size + : src1->nb[0] >= (size_t) src1->ne[1] * type_size); + + if (dim == 0 && + (dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16) && + dst->ne[0] <= max_gather_ne0 && + dst->nb[0] == type_size && dst->nb[1] == (size_t) dst->ne[0] * type_size && + src0->nb[0] == type_size && + (src1->nb[0] == type_size || src1->nb[1] == type_size) && + gather_strides_ok && + htp_gather_rows_tensor_outer_contiguous(dst) && + htp_gather_rows_tensor_outer_contiguous(src0) && + htp_gather_rows_tensor_outer_contiguous(src1)) { + + if (htp_gather_rows_solve_layout(&kparams->u.gather, + (uint32_t) src0->ne[0], (uint32_t) src0->nb[1], + (uint32_t) src1->ne[0], (uint32_t) src1->nb[1], b_dense, + type_size, total_rows, + sess->n_threads, sess->mdev.count, sess->vtcm_size)) { + kparams->kernel_type = HTP_CONCAT_KERNEL_GATHER_ROWS; + kparams->n_threads = (uint8_t) sess->n_threads; + kparams->vtcm_size = kparams->u.gather.slice * sess->n_threads; + return true; + } + } + const bool dma_strides_ok = (src0->nb[0] == type_size && src1->nb[0] == type_size && dst->nb[0] == type_size); if (dma_strides_ok) { @@ -6616,11 +6647,11 @@ static bool ggml_hexagon_precompute_concat_params( return false; } - kparams->kernel_type = HTP_CONCAT_KERNEL_TRANSPOSED; - kparams->n_threads = n_threads; - kparams->vtcm_size = layout.total_bytes; - kparams->spad0_size_per_thread = layout.src0_spad_size_per_thread; - kparams->spad1_size_per_thread = layout.src1_spad_size_per_thread; + kparams->kernel_type = HTP_CONCAT_KERNEL_TRANSPOSED; + kparams->n_threads = n_threads; + kparams->vtcm_size = layout.total_bytes; + kparams->u.transposed.spad0_size_per_thread = layout.src0_spad_size_per_thread; + kparams->u.transposed.spad1_size_per_thread = layout.src1_spad_size_per_thread; return true; } @@ -6706,6 +6737,27 @@ static bool ggml_hexagon_precompute_cpy_params( return true; } + const uint32_t max_gather_ne0 = 128 / src_type_size; + const uint32_t total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]); + if ((src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) && + dst_is_contiguous && !src_is_contiguous && + src0->nb[0] == src_type_size && src0->ne[0] <= max_gather_ne0 && + src0->nb[1] >= (size_t) src0->ne[0] * src_type_size && + total_rows > 1 && + htp_gather_rows_tensor_outer_contiguous(src0)) { + + if (htp_gather_rows_solve_layout(&kparams->u.gather, + (uint32_t) src0->ne[0], (uint32_t) src0->nb[1], + 0, 0, false, + src_type_size, total_rows, + sess->n_threads, sess->mdev.count, sess->vtcm_size)) { + kparams->kernel_type = HTP_COPY_KERNEL_GATHER_ROWS; + kparams->n_threads = (uint8_t) sess->n_threads; + kparams->vtcm_size = kparams->u.gather.slice * sess->n_threads; + return true; + } + } + if (sameshape) { kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_SAMETYPE; kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]); @@ -6737,7 +6789,7 @@ static bool ggml_hexagon_precompute_cpy_params( const uint32_t n_threads = sess->n_threads > 0 ? (uint32_t) sess->n_threads : 4; struct htp_copy_convert_vtcm_layout layout; - htp_copy_convert_vtcm_layout_build(&layout, (uint32_t) src0->ne[0], src_type_size, dst_type_size, n_threads); + htp_copy_convert_vtcm_layout_build(&layout, (uint32_t) src0->ne[0], (uint32_t) src0->ne[1], src_type_size, dst_type_size, n_threads); if (sess->vtcm_size > 0 && layout.total_bytes > sess->vtcm_size) { return false; @@ -6751,6 +6803,9 @@ static bool ggml_hexagon_precompute_cpy_params( kparams->u.convert.dst_buf_size = layout.dst_buf_size; kparams->u.convert.spad0_size_per_thread = layout.spad0_size_per_thread; kparams->u.convert.spad1_size_per_thread = layout.spad1_size_per_thread; + kparams->u.convert.src0_row_stride = layout.src0_row_stride; + kparams->u.convert.dst_row_stride = layout.dst_row_stride; + kparams->u.convert.blk_rows = layout.blk_rows; kparams->u.convert.div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]); kparams->u.convert.div_ne02_ne01 = init_fastdiv_values((uint32_t) (src0->ne[2] * src0->ne[1])); diff --git a/ggml/src/ggml-hexagon/htp/CMakeLists.txt b/ggml/src/ggml-hexagon/htp/CMakeLists.txt index 787badf07..bc1474ae8 100644 --- a/ggml/src/ggml-hexagon/htp/CMakeLists.txt +++ b/ggml/src/ggml-hexagon/htp/CMakeLists.txt @@ -33,6 +33,7 @@ add_library(${HTP_LIB} SHARED set-rows-ops.c get-rows-ops.c cpy-ops.c + gather-rows.c repeat-ops.c ssm-conv.c cumsum-ops.c diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.c b/ggml/src/ggml-hexagon/htp/concat-ops.c index f7dda1408..40fa09151 100644 --- a/ggml/src/ggml-hexagon/htp/concat-ops.c +++ b/ggml/src/ggml-hexagon/htp/concat-ops.c @@ -13,6 +13,7 @@ #include "htp-vtcm.h" #include "hvx-utils.h" #include "hvx_hexagon_protos.h" +#include "gather-rows.h" #include @@ -302,9 +303,9 @@ static int concat_transposed(struct htp_ops_context * octx, const struct htp_con struct htp_concat_context cctx; cctx.octx = octx; cctx.spad0_base = vtcm_base; - cctx.spad1_base = vtcm_base + n_threads * kparams->spad0_size_per_thread; - cctx.spad0_size_per_thread = kparams->spad0_size_per_thread; - cctx.spad1_size_per_thread = kparams->spad1_size_per_thread; + cctx.spad1_base = vtcm_base + n_threads * kparams->u.transposed.spad0_size_per_thread; + cctx.spad0_size_per_thread = kparams->u.transposed.spad0_size_per_thread; + cctx.spad1_size_per_thread = kparams->u.transposed.spad1_size_per_thread; cctx.row_start = row_start; cctx.nrows = nrows; cctx.nplanes = dst->ne[2] * dst->ne[3]; @@ -318,10 +319,13 @@ static int concat_transposed(struct htp_ops_context * octx, const struct htp_con int op_concat(struct htp_ops_context * octx) { const struct htp_concat_kernel_params * kparams = (const struct htp_concat_kernel_params *) octx->kernel_params; - const struct htp_tensor * dst = octx->dst; - const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2; + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * src1 = octx->src[1]; + const struct htp_tensor * dst = octx->dst; int status = HTP_STATUS_OK; + const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2; + switch (kparams->kernel_type) { case HTP_CONCAT_KERNEL_REGULAR: status = concat_regular(octx, kparams->dim, type_size); @@ -331,6 +335,10 @@ int op_concat(struct htp_ops_context * octx) { status = concat_transposed(octx, kparams, type_size); break; + case HTP_CONCAT_KERNEL_GATHER_ROWS: + status = htp_gather_rows(octx, src0, src1, &kparams->u.gather, kparams->n_threads); + break; + default: status = HTP_STATUS_NO_SUPPORT; break; @@ -338,9 +346,5 @@ int op_concat(struct htp_ops_context * octx) { htp_ops_context_set_status(octx, status); - if (octx->ctx->mdev.count > 1) { - htp_mdev_group_barrier(octx); - } - return octx->status; } diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.h b/ggml/src/ggml-hexagon/htp/concat-ops.h index 1d7218441..4bfba9894 100644 --- a/ggml/src/ggml-hexagon/htp/concat-ops.h +++ b/ggml/src/ggml-hexagon/htp/concat-ops.h @@ -2,12 +2,19 @@ #define HTP_CONCAT_OPS_H #include "hex-common.h" +#include "gather-rows.h" #include enum htp_concat_kernel_type { HTP_CONCAT_KERNEL_UNSUPPORTED = 0, HTP_CONCAT_KERNEL_REGULAR = 1, HTP_CONCAT_KERNEL_TRANSPOSED = 2, + HTP_CONCAT_KERNEL_GATHER_ROWS = 3, +}; + +struct htp_concat_transposed_params { + uint32_t spad0_size_per_thread; + uint32_t spad1_size_per_thread; }; struct htp_concat_kernel_params { @@ -17,8 +24,11 @@ struct htp_concat_kernel_params { uint8_t pad; uint32_t vtcm_size; - uint32_t spad0_size_per_thread; - uint32_t spad1_size_per_thread; + + union { + struct htp_concat_transposed_params transposed; + struct htp_gather_rows_params gather; + } u; }; #if defined(__cplusplus) diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.c b/ggml/src/ggml-hexagon/htp/cpy-ops.c index d366acb75..63ebb38e6 100644 --- a/ggml/src/ggml-hexagon/htp/cpy-ops.c +++ b/ggml/src/ggml-hexagon/htp/cpy-ops.c @@ -18,6 +18,7 @@ #include "htp-ops.h" #include "htp-tensor.h" #include "hvx-utils.h" +#include "gather-rows.h" struct htp_copy_context { struct htp_ops_context * octx; @@ -141,6 +142,25 @@ DEFINE_CPY_RESHAPE(f32, float, 4) DEFINE_CPY_RESHAPE(f16, __fp16, 2) DEFINE_CPY_RESHAPE(i32, int32_t, 4) +struct cpy_row_cursor { + uint32_t r; + uint32_t i01; + uint32_t i02; + uint32_t i03; +}; + +static inline void cpy_cursor_advance(struct cpy_row_cursor * c, uint32_t n, uint32_t ne01, uint32_t ne02) { + c->r += n; + c->i01 += n; + if (c->i01 == ne01) { + c->i01 = 0; + if (++c->i02 == ne02) { + c->i02 = 0; + c->i03++; + } + } +} + #define DEFINE_CPY_CONVERT_SAMESHAPE(NAME, CONV_FUNC) \ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \ struct htp_copy_context * ct = (struct htp_copy_context *) data; \ @@ -151,7 +171,6 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vo const uint32_t ir0 = ct->row_start + dr * ith; \ const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); \ if (ir0 >= ir1) return; \ - const uint32_t nrows_thread = ir1 - ir0; \ \ dma_queue * dma_q = octx->ctx->dma[ith]; \ struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ @@ -159,6 +178,9 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vo const struct htp_copy_convert_params * cvt = &ct->kparams->u.convert; \ const uint32_t src0_buf_size = cvt->src0_buf_size; \ const uint32_t dst_buf_size = cvt->dst_buf_size; \ + const uint32_t src0_row_stride = cvt->src0_row_stride; \ + const uint32_t dst_row_stride = cvt->dst_row_stride; \ + const uint32_t blk_rows = cvt->blk_rows; \ uint8_t * vtcm_src0_base = ct->vtcm_src0 + ith * cvt->spad0_size_per_thread; \ uint8_t * vtcm_dst_base = ct->vtcm_dst + ith * cvt->spad1_size_per_thread; \ const uint32_t src0_row_size = ne00 * ct->kparams->src0_type_size; \ @@ -170,62 +192,43 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vo uint32_t i02 = fastdiv(rem, &cvt->div_ne01); \ uint32_t i01 = rem - i02 * ne01; \ \ - uint32_t f_i01 = i01, f_i02 = i02, f_i03 = i03; \ - dma_addr_t f_src0_addr = src0->data + f_i01*nb01 + f_i02*nb02 + f_i03*nb03; \ + struct cpy_row_cursor f = { ir0, i01, i02, i03 }; \ + struct cpy_row_cursor c = f; \ \ - uint32_t c_i01 = i01, c_i02 = i02, c_i03 = i03; \ - dma_addr_t c_dst_addr = dst->data + c_i01*nb1 + c_i02*nb2 + c_i03*nb3; \ - \ - for (uint32_t r = 0; r < nrows_thread && r < 2; ++r) { \ - uint8_t * src_spad = vtcm_src0_base + r * src0_buf_size; \ - uint8_t * dst_spad = vtcm_dst_base + r * dst_buf_size; \ + for (uint32_t b = 0; b < 2 && f.r < ir1; ++b) { \ + uint8_t * src_spad = vtcm_src0_base + b * src0_buf_size; \ + uint8_t * dst_spad = vtcm_dst_base + b * dst_buf_size; \ + const uint32_t nr = MIN(blk_rows, MIN(ne01 - f.i01, ir1 - f.r)); \ + dma_addr_t f_src0_addr = src0->data + f.i01*nb01 + f.i02*nb02 + f.i03*nb03; \ dma_queue_push(dma_q, dma_make_data(dst->data, dst_spad), \ dst_row_size, dst_buf_size, dst_row_size, 0); \ dma_queue_push(dma_q, dma_make_data(src_spad, f_src0_addr), \ - src0_buf_size, src0_row_size, src0_row_size, 1); \ - f_src0_addr += nb01; \ - if (++f_i01 == ne01) { \ - f_i01 = 0; \ - if (++f_i02 == ne02) { \ - f_i02 = 0; \ - f_i03++; \ - } \ - f_src0_addr = src0->data + f_i02*nb02 + f_i03*nb03; \ - } \ + src0_row_stride, nb01, src0_row_size, nr); \ + cpy_cursor_advance(&f, nr, ne01, ne02); \ } \ \ - for (uint32_t r = 0; r < nrows_thread; ++r) { \ + for (uint32_t blk = 0; c.r < ir1; ++blk) { \ + const uint32_t nr = MIN(blk_rows, MIN(ne01 - c.i01, ir1 - c.r)); \ uint8_t * dst_spad = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).src; \ uint8_t * src_spad = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).dst; \ \ - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \ - CONV_FUNC(dst_spad, src_spad, ne00); \ - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \ - \ - dma_queue_push(dma_q, dma_make_data(c_dst_addr, dst_spad), \ - dst_row_size, dst_buf_size, dst_row_size, 1); \ - c_dst_addr += nb1; \ - if (++c_i01 == ne01) { \ - c_i01 = 0; \ - if (++c_i02 == ne02) { \ - c_i02 = 0; \ - c_i03++; \ - } \ - c_dst_addr = dst->data + c_i02*nb2 + c_i03*nb3; \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) blk); \ + for (uint32_t r = 0; r < nr; ++r) { \ + CONV_FUNC(dst_spad + r * dst_row_stride, src_spad + r * src0_row_stride, ne00); \ } \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) blk); \ \ - if (r + 2 < nrows_thread) { \ + dma_addr_t c_dst_addr = dst->data + c.i01*nb1 + c.i02*nb2 + c.i03*nb3; \ + dma_queue_push(dma_q, dma_make_data(c_dst_addr, dst_spad), \ + nb1, dst_row_stride, dst_row_size, nr); \ + cpy_cursor_advance(&c, nr, ne01, ne02); \ + \ + if (f.r < ir1) { \ + const uint32_t fnr = MIN(blk_rows, MIN(ne01 - f.i01, ir1 - f.r)); \ + dma_addr_t f_src0_addr = src0->data + f.i01*nb01 + f.i02*nb02 + f.i03*nb03; \ dma_queue_push(dma_q, dma_make_data(src_spad, f_src0_addr), \ - src0_buf_size, src0_row_size, src0_row_size, 1); \ - f_src0_addr += nb01; \ - if (++f_i01 == ne01) { \ - f_i01 = 0; \ - if (++f_i02 == ne02) { \ - f_i02 = 0; \ - f_i03++; \ - } \ - f_src0_addr = src0->data + f_i02*nb02 + f_i03*nb03; \ - } \ + src0_row_stride, nb01, src0_row_size, fnr); \ + cpy_cursor_advance(&f, fnr, ne01, ne02); \ } \ } \ dma_queue_flush(dma_q); \ @@ -337,10 +340,6 @@ static int cpy_sameshape_convert(struct htp_ops_context * octx, const struct htp const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * dst = octx->dst; - if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) { - return HTP_STATUS_NO_SUPPORT; - } - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { return HTP_STATUS_INVAL_PARAMS; } @@ -398,10 +397,6 @@ static int cpy_reshape(struct htp_ops_context * octx, const struct htp_copy_kern const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * dst = octx->dst; - if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) { - return HTP_STATUS_NO_SUPPORT; - } - if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { return HTP_STATUS_INVAL_PARAMS; } @@ -461,6 +456,9 @@ int op_cpy(struct htp_ops_context * octx) { case HTP_COPY_KERNEL_RESHAPE: status = cpy_reshape(octx, kparams); break; + case HTP_COPY_KERNEL_GATHER_ROWS: + status = htp_gather_rows(octx, octx->src[0], NULL, &kparams->u.gather, kparams->n_threads); + break; default: status = HTP_STATUS_NO_SUPPORT; break; @@ -468,11 +466,11 @@ int op_cpy(struct htp_ops_context * octx) { htp_ops_context_set_status(octx, status); - if (octx->ctx->mdev.count > 1) { - htp_mdev_group_barrier(octx); - } - if (octx->op == HTP_OP_CPY_FENCE) { + if (octx->ctx->mdev.count > 1) { + htp_mdev_group_barrier(octx); + } + if (octx->ctx->mdev.idx == 0) { const struct htp_tensor * sync = octx->src[1]; const uint32_t seq = (uint32_t) octx->op_params[0]; diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.h b/ggml/src/ggml-hexagon/htp/cpy-ops.h index 4cf7dc13b..16695c2ab 100644 --- a/ggml/src/ggml-hexagon/htp/cpy-ops.h +++ b/ggml/src/ggml-hexagon/htp/cpy-ops.h @@ -3,6 +3,7 @@ #include "hex-common.h" #include "hex-fastdiv.h" +#include "gather-rows.h" #include enum htp_copy_kernel_type { @@ -12,6 +13,7 @@ enum htp_copy_kernel_type { HTP_COPY_KERNEL_SAMESHAPE_CONVERT = 3, HTP_COPY_KERNEL_RESHAPE = 4, HTP_COPY_KERNEL_SCALAR = 5, + HTP_COPY_KERNEL_GATHER_ROWS = 6, }; struct htp_copy_convert_params { @@ -19,6 +21,9 @@ struct htp_copy_convert_params { uint32_t dst_buf_size; uint32_t spad0_size_per_thread; uint32_t spad1_size_per_thread; + uint32_t src0_row_stride; // row pitch in the VTCM buffers + uint32_t dst_row_stride; + uint32_t blk_rows; // rows moved per DMA descriptor struct fastdiv_values div_ne01; struct fastdiv_values div_ne02_ne01; }; @@ -45,10 +50,19 @@ struct htp_copy_kernel_params { union { struct htp_copy_convert_params convert; struct htp_copy_reshape_params reshape; + struct htp_gather_rows_params gather; } u; }; +// Size of one VTCM staging buffer: big enough that a DMA descriptor moves many short +// rows at once. The copy kernels keep four per thread (two in flight each way), about +// 1 MB of VTCM with 8 threads. +#define HTP_COPY_VTCM_BLK_BYTES (32 * 1024) + struct htp_copy_convert_vtcm_layout { + uint32_t src0_row_stride; + uint32_t dst_row_stride; + uint32_t blk_rows; uint32_t src0_buf_size; uint32_t dst_buf_size; uint32_t spad0_size_per_thread; @@ -56,15 +70,30 @@ struct htp_copy_convert_vtcm_layout { uint32_t total_bytes; }; +// ne01 bounds a block: rows of one block share the dim-1 stride. static inline void htp_copy_convert_vtcm_layout_build( struct htp_copy_convert_vtcm_layout * layout, uint32_t ne00, + uint32_t ne01, uint32_t src_type_size, uint32_t dst_type_size, uint32_t n_threads) { - layout->src0_buf_size = hex_round_up(ne00 * src_type_size, 256); - layout->dst_buf_size = hex_round_up(ne00 * dst_type_size, 256); + layout->src0_row_stride = hex_round_up(ne00 * src_type_size, 256); + layout->dst_row_stride = hex_round_up(ne00 * dst_type_size, 256); + + const uint32_t max_row = layout->src0_row_stride > layout->dst_row_stride ? layout->src0_row_stride : layout->dst_row_stride; + uint32_t blk_rows = HTP_COPY_VTCM_BLK_BYTES / max_row; + if (blk_rows > ne01) { + blk_rows = ne01; + } + if (blk_rows == 0) { + blk_rows = 1; + } + layout->blk_rows = blk_rows; + + layout->src0_buf_size = blk_rows * layout->src0_row_stride; + layout->dst_buf_size = blk_rows * layout->dst_row_stride; layout->spad0_size_per_thread = 2 * layout->src0_buf_size; layout->spad1_size_per_thread = 2 * layout->dst_buf_size; layout->total_bytes = n_threads * (layout->spad0_size_per_thread + layout->spad1_size_per_thread); diff --git a/ggml/src/ggml-hexagon/htp/gather-rows.c b/ggml/src/ggml-hexagon/htp/gather-rows.c new file mode 100644 index 000000000..1db48e59c --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/gather-rows.c @@ -0,0 +1,387 @@ +#include "gather-rows.h" + +#include + +#include "hex-common.h" +#include "hex-profile.h" +#include "hvx-utils.h" +#include "dma-queue.h" +#include "htp-ctx.h" +#include "htp-ops.h" +#include "htp-tensor.h" +#include "work-queue.h" + +typedef void (*gather_rows_compute_fn_t)( + uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec, + const struct htp_gather_rows_params * restrict kparams, + const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur); + +struct gather_rows_task { + struct htp_ops_context * octx; + const struct htp_tensor * a; + const struct htp_tensor * b; + dma_addr_t dst; + uint32_t r_start; + uint32_t rows; + uint32_t rows_per_thread; + const struct htp_gather_rows_params * kparams; + gather_rows_compute_fn_t compute; +}; + +static inline __attribute__((always_inline)) void gather_rows_dma_in( + dma_queue * q, const struct gather_rows_task * task, + uint8_t * region, uint32_t r, uint32_t nr, uint32_t elem_size) { + const struct htp_tensor * a = task->a; + const struct htp_tensor * b = task->b; + const struct htp_gather_rows_params * kparams = task->kparams; + + const size_t a_size = (nr - 1) * a->nb[1] + kparams->na * elem_size; + dma_queue_push(q, dma_make_data(region, a->data + r * a->nb[1]), a_size, a_size, a_size, 1); + if (b) { + if (kparams->b_dense) { + const size_t b_size = (nr - 1) * b->nb[1] + kparams->nb * elem_size; + dma_queue_push(q, dma_make_data(region + kparams->v1, b->data + r * b->nb[1]), b_size, b_size, b_size, 1); + } else { + const size_t b_size = (nr - 1) * b->nb[1] + elem_size; + for (uint32_t k = 0; k < kparams->nb; k++) { + dma_queue_push(q, dma_make_data(region + kparams->v1 + k * kparams->span1, + b->data + k * b->nb[0] + r * b->nb[1]), b_size, b_size, b_size, 1); + } + } + } +} + +static inline void gather_rows_dma_out(dma_queue * q, const struct gather_rows_task * task, + uint8_t * out, uint32_t r, uint32_t nr, uint32_t elem_size) { + const struct htp_gather_rows_params * kparams = task->kparams; + const size_t out_size = nr * kparams->ne * elem_size; + dma_queue_push(q, dma_make_data(task->dst + r * kparams->ne * elem_size, out), out_size, out_size, out_size, 1); +} + +static __attribute__((noinline)) void gather_rows_init_tables_f32( + HVX_Vector * tab, HVX_Vector * inc, + const struct htp_gather_rows_params * kparams, + const struct htp_tensor * a, const struct htp_tensor * b) { + + const int32_t s_a = kparams->rg * a->nb[1]; + const int32_t s_b = b ? kparams->rg * b->nb[1] : 0; + const uint32_t b_nb1 = b ? b->nb[1] : 0; + uint32_t row = 0; + uint32_t col = 0; + int32_t t_buf[32] __attribute__((aligned(128))); + int32_t s_buf[32] __attribute__((aligned(128))); + for (uint32_t k = 0; k < kparams->vg; k++) { + for (uint32_t lane = 0; lane < 32; lane++) { + if (col < kparams->na) { + t_buf[lane] = row * a->nb[1] + col * 4; + s_buf[lane] = s_a; + } else { + t_buf[lane] = kparams->v1 + (col - kparams->na) * kparams->span1 + row * b_nb1; + s_buf[lane] = s_b; + } + if (++col == kparams->ne) { + col = 0; + row++; + } + } + tab[k] = *(const HVX_Vector *) t_buf; + inc[k] = *(const HVX_Vector *) s_buf; + } +} + +static __attribute__((noinline)) void gather_rows_init_tables_f16( + HVX_Vector * tab, HVX_Vector * inc, + const struct htp_gather_rows_params * kparams, + const struct htp_tensor * a, const struct htp_tensor * b) { + + const int16_t s_a = (int16_t) (kparams->rg * a->nb[1]); + const int16_t s_b = (int16_t) (b ? kparams->rg * b->nb[1] : 0); + const uint32_t b_nb1 = b ? b->nb[1] : 0; + uint32_t row = 0; + uint32_t col = 0; + int16_t t_buf[64] __attribute__((aligned(128))); + int16_t s_buf[64] __attribute__((aligned(128))); + for (uint32_t k = 0; k < kparams->vg; k++) { + for (uint32_t lane = 0; lane < 64; lane++) { + if (col < kparams->na) { + t_buf[lane] = (int16_t) (row * a->nb[1] + col * 2); + s_buf[lane] = s_a; + } else { + t_buf[lane] = (int16_t) (kparams->v1 + (col - kparams->na) * kparams->span1 + row * b_nb1); + s_buf[lane] = s_b; + } + if (++col == kparams->ne) { + col = 0; + row++; + } + } + tab[k] = *(const HVX_Vector *) t_buf; + inc[k] = *(const HVX_Vector *) s_buf; + } +} + +static void gather_rows_compute_f32_vg1( + uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec, + const struct htp_gather_rows_params * restrict kparams, + const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) { + (void) cur; + const uint32_t region_size = kparams->region_size; + HVX_Vector vcur = tab[0]; + const HVX_Vector vinc = inc[0]; + #pragma unroll(4) + for (uint32_t v = 0; v < nvec; v++) { + Q6_vgather_ARMVw((HVX_Vector *) (out + v * 128), (size_t) region, region_size, vcur); + vcur = Q6_Vw_vadd_VwVw(vcur, vinc); + } +} + +static void gather_rows_compute_f32_vgn( + uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec, + const struct htp_gather_rows_params * restrict kparams, + const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) { + const uint32_t vg = kparams->vg; + const uint32_t region_size = kparams->region_size; + for (uint32_t k = 0; k < vg; k++) { + cur[k] = tab[k]; + } + uint32_t k = 0; + #pragma unroll(4) + for (uint32_t v = 0; v < nvec; v++) { + Q6_vgather_ARMVw((HVX_Vector *) (out + v * 128), (size_t) region, region_size, cur[k]); + cur[k] = Q6_Vw_vadd_VwVw(cur[k], inc[k]); + if (++k == vg) { + k = 0; + } + } +} + +static void gather_rows_compute_f16_vg1( + uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec, + const struct htp_gather_rows_params * restrict kparams, + const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) { + (void) cur; + const uint32_t region_size = kparams->region_size; + HVX_Vector vcur = tab[0]; + const HVX_Vector vinc = inc[0]; + #pragma unroll(4) + for (uint32_t v = 0; v < nvec; v++) { + Q6_vgather_ARMVh((HVX_Vector *) (out + v * 128), (size_t) region, region_size, vcur); + vcur = Q6_Vh_vadd_VhVh(vcur, vinc); + } +} + +static void gather_rows_compute_f16_vgn( + uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec, + const struct htp_gather_rows_params * restrict kparams, + const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) { + const uint32_t vg = kparams->vg; + const uint32_t region_size = kparams->region_size; + for (uint32_t k = 0; k < vg; k++) { + cur[k] = tab[k]; + } + uint32_t k = 0; + #pragma unroll(4) + for (uint32_t v = 0; v < nvec; v++) { + Q6_vgather_ARMVh((HVX_Vector *) (out + v * 128), (size_t) region, region_size, cur[k]); + cur[k] = Q6_Vh_vadd_VhVh(cur[k], inc[k]); + if (++k == vg) { + k = 0; + } + } +} + +static void gather_rows_thread_f32(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + const struct gather_rows_task * task = (const struct gather_rows_task *) data; + const struct htp_gather_rows_params * kparams = task->kparams; + const struct htp_tensor * a = task->a; + const struct htp_tensor * b = task->b; + + const uint32_t r_offset = MIN(ith * task->rows_per_thread, task->rows); + const uint32_t r_beg = task->r_start + r_offset; + const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_thread, task->rows); + if (r_beg >= r_end) { + return; + } + + struct htp_thread_trace * tr = &task->octx->ctx->trace[ith]; + + uint8_t * base = task->octx->ctx->vtcm_base + ith * kparams->slice; + HVX_Vector * tab = (HVX_Vector *) base; + HVX_Vector * inc = tab + kparams->vg; + HVX_Vector * cur = inc + kparams->vg; + uint8_t * region_base = (uint8_t *) (cur + kparams->vg); + + uint8_t * region[2] = { region_base, region_base + kparams->buf_stride }; + uint8_t * out[2] = { region[0] + kparams->out_off, region[1] + kparams->out_off }; + + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg); + gather_rows_init_tables_f32(tab, inc, kparams, a, b); + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg); + + dma_queue * q = task->octx->ctx->dma[ith]; + const uint32_t n_in = 1 + (task->b ? (kparams->b_dense ? 1 : kparams->nb) : 0); + const uint32_t nr_max = kparams->nr_max; + const uint32_t nr0 = MIN(nr_max, r_end - r_beg); + + dma_queue_push(q, dma_make_data(task->dst, out[0]), 0, 0, 0, 0); // dummy out + gather_rows_dma_in(q, task, region[0], r_beg, nr0, 4); + + const uint32_t r1 = r_beg + nr_max; + if (r1 < r_end) { + const uint32_t nr1 = MIN(nr_max, r_end - r1); + dma_queue_push(q, dma_make_data(task->dst, out[1]), 0, 0, 0, 0); // dummy out + gather_rows_dma_in(q, task, region[1], r1, nr1, 4); + } + + uint32_t buf = 0; + for (uint32_t r = r_beg; r < r_end; r += nr_max) { + const uint32_t nr = MIN(nr_max, r_end - r); + const uint32_t next_buf = buf ^ 1; + + dma_queue_pop(q); // complete out + for (uint32_t i = 0; i < n_in; i++) { + dma_queue_pop(q); + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); + const uint32_t nvec = (nr * kparams->ne + 31) >> 5; + task->compute(out[buf], region[buf], nvec, kparams, tab, inc, cur); + hvx_gather_sync(out[buf]); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); + + gather_rows_dma_out(q, task, out[buf], r, nr, 4); + + const uint32_t r_prefetch = r + 2 * nr_max; + if (r_prefetch < r_end) { + const uint32_t nr_prefetch = MIN(nr_max, r_end - r_prefetch); + gather_rows_dma_in(q, task, region[buf], r_prefetch, nr_prefetch, 4); + } + + buf = next_buf; + } + dma_queue_flush(q); +} + +static void gather_rows_thread_f16(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + const struct gather_rows_task * task = (const struct gather_rows_task *) data; + const struct htp_gather_rows_params * kparams = task->kparams; + const struct htp_tensor * a = task->a; + const struct htp_tensor * b = task->b; + + const uint32_t r_offset = MIN(ith * task->rows_per_thread, task->rows); + const uint32_t r_beg = task->r_start + r_offset; + const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_thread, task->rows); + if (r_beg >= r_end) { + return; + } + + struct htp_thread_trace * tr = &task->octx->ctx->trace[ith]; + + uint8_t * base = task->octx->ctx->vtcm_base + ith * kparams->slice; + HVX_Vector * tab = (HVX_Vector *) base; + HVX_Vector * inc = tab + kparams->vg; + HVX_Vector * cur = inc + kparams->vg; + uint8_t * region_base = (uint8_t *) (cur + kparams->vg); + + uint8_t * region[2] = { region_base, region_base + kparams->buf_stride }; + uint8_t * out[2] = { region[0] + kparams->out_off, region[1] + kparams->out_off }; + + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg); + gather_rows_init_tables_f16(tab, inc, kparams, a, b); + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg); + + dma_queue * q = task->octx->ctx->dma[ith]; + const uint32_t n_in = 1 + (task->b ? (kparams->b_dense ? 1 : kparams->nb) : 0); + + const uint32_t nr_max = kparams->nr_max; + const uint32_t nr0 = MIN(nr_max, r_end - r_beg); + dma_queue_push(q, dma_make_data(task->dst, out[0]), 0, 0, 0, 0); // dummy out + gather_rows_dma_in(q, task, region[0], r_beg, nr0, 2); + + const uint32_t r1 = r_beg + nr_max; + if (r1 < r_end) { + const uint32_t nr1 = MIN(nr_max, r_end - r1); + dma_queue_push(q, dma_make_data(task->dst, out[1]), 0, 0, 0, 0); // dummy out + gather_rows_dma_in(q, task, region[1], r1, nr1, 2); + } + + uint32_t buf = 0; + for (uint32_t r = r_beg; r < r_end; r += nr_max) { + const uint32_t nr = MIN(nr_max, r_end - r); + const uint32_t next_buf = buf ^ 1; + + dma_queue_pop(q); // complete out + for (uint32_t i = 0; i < n_in; i++) { + dma_queue_pop(q); + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); + const uint32_t nvec = (nr * kparams->ne + 63) >> 6; + task->compute(out[buf], region[buf], nvec, kparams, tab, inc, cur); + hvx_gather_sync(out[buf]); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); + + gather_rows_dma_out(q, task, out[buf], r, nr, 2); + + const uint32_t r_prefetch = r + 2 * nr_max; + if (r_prefetch < r_end) { + const uint32_t nr_prefetch = MIN(nr_max, r_end - r_prefetch); + gather_rows_dma_in(q, task, region[buf], r_prefetch, nr_prefetch, 2); + } + + buf = next_buf; + } + dma_queue_flush(q); +} + +int htp_gather_rows( + struct htp_ops_context * octx, + const struct htp_tensor * a, + const struct htp_tensor * b, + const struct htp_gather_rows_params * kparams, + uint32_t n_threads) { + + if (!htp_ops_context_set_n_threads(octx, n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + assert(kparams->slice * octx->n_threads <= octx->ctx->vtcm_size); + + const uint32_t total_rows = kparams->total_rows; + uint32_t r_start = 0; + uint32_t r_count = total_rows; + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + total_rows, kparams->rg, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + r_start = range.start; + r_count = range.count; + } + + if (r_count == 0) { + return HTP_STATUS_OK; + } + + const uint32_t elem_size = (a->type == HTP_TYPE_F16) ? 2 : 4; + + struct gather_rows_task task; + task.octx = octx; + task.a = a; + task.b = b; + task.dst = octx->dst->data; + task.r_start = r_start; + task.rows = r_count; + task.kparams = kparams; + task.rows_per_thread = kparams->rows_per_thread; + + if (elem_size == 2) { + task.compute = (kparams->vg == 1) ? gather_rows_compute_f16_vg1 : gather_rows_compute_f16_vgn; + } else { + task.compute = (kparams->vg == 1) ? gather_rows_compute_f32_vg1 : gather_rows_compute_f32_vgn; + } + + work_queue_func_t worker_func = (elem_size == 2) ? gather_rows_thread_f16 : gather_rows_thread_f32; + work_queue_run(octx->ctx->work_queue, worker_func, &task, octx->n_threads); + return HTP_STATUS_OK; +} diff --git a/ggml/src/ggml-hexagon/htp/gather-rows.h b/ggml/src/ggml-hexagon/htp/gather-rows.h new file mode 100644 index 000000000..3bbb1c635 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/gather-rows.h @@ -0,0 +1,160 @@ +#ifndef HTP_GATHER_ROWS_H +#define HTP_GATHER_ROWS_H + +#include "hex-common.h" +#include +#include + +#ifndef GGML_MAX_DIMS +#define GGML_MAX_DIMS 4 +#endif + +struct htp_gather_rows_params { + uint32_t na; + uint32_t nb; + uint32_t ne; + uint32_t rg; + uint32_t vg; + uint32_t nr_max; + uint32_t v1; + uint32_t span1; + uint32_t region_size; + uint32_t out_off; + uint32_t buf_stride; + uint32_t slice; + uint32_t total_rows; + uint32_t rows_per_thread; + uint8_t b_dense; + uint8_t pad[3]; +}; + +static inline bool htp_gather_rows_solve_layout( + struct htp_gather_rows_params * kparams, + uint32_t na, uint32_t a_nb1, + uint32_t nb, uint32_t b_nb1, bool b_dense, + uint32_t elem_size, + uint32_t total_rows, + uint32_t n_threads, + uint32_t mdev_count, + uint32_t vtcm_size) { + + const uint32_t max_ne0 = 128 / elem_size; + const uint32_t ne = na + nb; + if (ne == 0 || ne > max_ne0) { + return false; + } + + if (elem_size != 4 && elem_size != 2) { + return false; + } + + const uint32_t v_lanes = 128 / elem_size; + const uint32_t g = hex_gcd_u32(ne, v_lanes); + const uint32_t rg = v_lanes / g; + const uint32_t vg = ne / g; + + const uint32_t slice = (vtcm_size / n_threads) & ~127u; + const uint32_t total_chunks = total_rows / rg; + const uint32_t chunks_per_dev = (total_chunks + mdev_count - 1) / mdev_count; + const uint32_t max_rows_per_dev = (mdev_count > 1 && total_chunks >= mdev_count) + ? (chunks_per_dev * rg + (total_rows % rg)) + : total_rows; + const uint32_t rows_per_thread = hex_round_up((max_rows_per_dev + n_threads - 1) / n_threads, rg); + + const uint32_t fixed = 3 * vg * 128; + const uint32_t per_row = a_nb1 + (nb > 0 ? (b_dense ? b_nb1 : nb * b_nb1) : 0) + ne * elem_size; + if (slice <= fixed + 2 * per_row * rg) { + return false; + } + + const uint32_t avail_per_buf = (slice - fixed) / 2; + uint32_t nr_max = MIN(avail_per_buf / per_row, rows_per_thread); + if (elem_size == 2) { + const uint32_t per_row_region = a_nb1 + (nb > 0 ? (b_dense ? b_nb1 : nb * b_nb1) : 0); + if (per_row_region > 0) { + nr_max = MIN(nr_max, (32768u - 256u) / per_row_region); + } + } + nr_max = (nr_max / rg) * rg; + + uint32_t v1 = 0; + uint32_t span1 = 0; + uint32_t region_size = 0; + uint32_t out_off = 0; + uint32_t buf_stride = 0; + + while (nr_max > 0) { + v1 = hex_round_up((nr_max - 1) * a_nb1 + na * elem_size, 128); + if (nb > 0) { + if (b_dense) { + span1 = elem_size; + region_size = v1 + (nr_max - 1) * b_nb1 + nb * elem_size; + } else { + span1 = (nr_max - 1) * b_nb1 + elem_size; + region_size = v1 + nb * span1; + } + } else { + span1 = 0; + region_size = v1; + } + + out_off = hex_round_up(region_size, 128); + buf_stride = hex_round_up(out_off + nr_max * ne * elem_size, 128); + + if ((elem_size != 2 || region_size <= 32768) && fixed + 2 * buf_stride <= slice) { + break; + } + + nr_max -= rg; + } + + if (nr_max == 0) { + return false; + } + + kparams->na = na; + kparams->nb = nb; + kparams->ne = ne; + kparams->rg = rg; + kparams->vg = vg; + kparams->nr_max = nr_max; + kparams->v1 = v1; + kparams->span1 = span1; + kparams->region_size = region_size; + kparams->out_off = out_off; + kparams->buf_stride = buf_stride; + kparams->slice = slice; + kparams->total_rows = total_rows; + kparams->rows_per_thread = rows_per_thread; + kparams->b_dense = b_dense ? 1 : 0; + kparams->pad[0] = 0; + kparams->pad[1] = 0; + kparams->pad[2] = 0; + return true; +} + +#ifdef __cplusplus +struct ggml_tensor; +static inline bool htp_gather_rows_tensor_outer_contiguous(const struct ggml_tensor * t) { + size_t next_nb = ((const size_t *) t->nb)[1] * ((const int64_t *) t->ne)[1]; + for (int i = 2; i < GGML_MAX_DIMS; i++) { + if (((const int64_t *) t->ne)[i] != 1 && ((const size_t *) t->nb)[i] != next_nb) { + return false; + } + next_nb *= ((const int64_t *) t->ne)[i]; + } + return true; +} +#else +struct htp_ops_context; +struct htp_tensor; + +int htp_gather_rows( + struct htp_ops_context * octx, + const struct htp_tensor * a, + const struct htp_tensor * b, + const struct htp_gather_rows_params * kparams, + uint32_t n_threads); +#endif + +#endif // HTP_GATHER_ROWS_H diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index 6e3482b91..aff1c6c0f 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -287,15 +287,15 @@ AEEResult htp_iface_munmap(remote_handle64 handle, uint32 fd) { } static void vtcm_acquire(struct htp_context * ctx) { - if (!ctx->vtcm_valid) { + if (!atomic_load(&ctx->vtcm_valid)) { int err = HAP_compute_res_acquire_cached(ctx->vtcm_rctx, 10000000u); if (err != 0) { FARF(ERROR, "ggml-hex: failed to acquire VTCM: 0x%08x", (unsigned)err); abort(); } - ctx->vtcm_needs_release = false; - ctx->vtcm_valid = true; + atomic_store(&ctx->vtcm_needs_release, false); + atomic_store(&ctx->vtcm_valid, true); // Drop the priority to make sure we get the release callback from other GGML-HTP and QNN-HTP sessions HAP_compute_res_update_priority(ctx->vtcm_rctx, ctx->thread_prio + 10); @@ -303,16 +303,16 @@ static void vtcm_acquire(struct htp_context * ctx) { } static void vtcm_release(struct htp_context * ctx) { - if (ctx->vtcm_valid) { - ctx->vtcm_valid = false; - ctx->vtcm_needs_release = false; + if (atomic_load(&ctx->vtcm_valid)) { + atomic_store(&ctx->vtcm_valid, false); + atomic_store(&ctx->vtcm_needs_release, false); HAP_compute_res_release_cached(ctx->vtcm_rctx); } } static int vtcm_release_callback(unsigned int rctx, void * state) { struct htp_context * ctx = (struct htp_context *) state; - ctx->vtcm_needs_release = true; + atomic_store(&ctx->vtcm_needs_release, true); return 0; } @@ -345,8 +345,8 @@ static int vtcm_alloc(struct htp_context * ctx) { ctx->vtcm_base = (uint8_t *) vtcm_ptr; ctx->vtcm_size = vtcm_size; ctx->vtcm_rctx = rctx; - ctx->vtcm_valid = false; - ctx->vtcm_needs_release = false; + atomic_store(&ctx->vtcm_valid, false); + atomic_store(&ctx->vtcm_needs_release, false); return 0; } @@ -1309,7 +1309,7 @@ static void process_ops(struct htp_context * ctx) { vtcm_acquire(ctx); - while (!ctx->vtcm_needs_release && !atomic_load(&ctx->killed)) { + while (!atomic_load(&ctx->vtcm_needs_release) && !atomic_load(&ctx->killed)) { struct htp_opbatch_req req; uint32_t r_size = sizeof(req); diff --git a/scripts/snapdragon/build.py b/scripts/snapdragon/build.py index b36e6361b..62a114e04 100755 --- a/scripts/snapdragon/build.py +++ b/scripts/snapdragon/build.py @@ -10,6 +10,7 @@ import subprocess import platform import shutil import logging +import glob from sdk import validate_windows_sdks @@ -51,6 +52,8 @@ def main(): parser.add_argument("--no-docker", action="store_true", help="Build natively on the host instead of in a docker container") parser.add_argument("--preset", help="Override the CMake preset to use") parser.add_argument("--debug", action="store_true", help="Build in debug mode (uses -debug presets instead of -release)") + parser.add_argument("--hex-only", "--hexagon-only", action="store_true", help="Only build and push ggml-hexagon and libggml-htp libraries") + parser.add_argument("--hex-arch", choices=["v73", "v75", "v79", "v81", "73", "75", "79", "81"], help="Specific Hexagon DSP architecture to build with --hex-only (default: all)") # Push options parser.add_argument("--push", action="store_true", help="Push built package to the target device via ADB or SSH/SCP") @@ -129,6 +132,16 @@ def main(): jobs = args.jobs if args.jobs else os.cpu_count() or 4 + hex_arch = None + if args.hex_arch: + hex_arch = args.hex_arch if args.hex_arch.startswith("v") else f"v{args.hex_arch}" + + if args.hex_only: + if hex_arch: + hex_targets = ["ggml-hexagon", f"htp-{hex_arch}"] + else: + hex_targets = ["ggml-hexagon", "htp-v73", "htp-v75", "htp-v79", "htp-v81"] + if args.no_docker: # Native/local host build logger.info("Running native/local CMake build...") @@ -143,7 +156,10 @@ def main(): sys.exit(res.returncode) # Build - build_cmd = ["cmake", "--build", build_dir, "-j", str(jobs)] + build_cmd = ["cmake", "--build", build_dir] + if args.hex_only: + build_cmd += ["--target"] + hex_targets + build_cmd += ["-j", str(jobs)] logger.info(f"+ {' '.join(build_cmd)}") res = subprocess.run(build_cmd, cwd=repo_root) if res.returncode != 0: @@ -165,9 +181,10 @@ def main(): install_prefix_container = f"/workspace/{install_dir}/llama.cpp" + target_args = f"--target {' '.join(hex_targets)} " if args.hex_only else "" build_sh_cmd = ( f"cmake --preset {preset} -B /workspace/{build_dir} && " - f"cmake --build /workspace/{build_dir} -j {jobs} && " + f"cmake --build /workspace/{build_dir} {target_args}-j {jobs} && " f"cmake --install /workspace/{build_dir} --prefix {install_prefix_container}" ) @@ -206,58 +223,91 @@ def main(): sub_items = [item for item in os.listdir(src_path) if not item.startswith(".")] + if args.hex_only: + lib_dir = os.path.join(src_path, "lib") + hex_libs = glob.glob(os.path.join(lib_dir, "libggml-hexagon*")) + if hex_arch: + hex_libs += glob.glob(os.path.join(lib_dir, f"libggml-htp-{hex_arch}*")) + else: + hex_libs += glob.glob(os.path.join(lib_dir, "libggml-htp*")) + + if not hex_libs: + logger.error(f"Error: no hexagon libraries found in {lib_dir} to push.") + sys.exit(1) + if target_type == "android": - logger.info("\nPushing built artifacts to Android device via ADB...") adb_cmd = ["adb"] if target_val: # serial adb_cmd += ["-s", target_val] - # Clean stale package files on device - if sub_items: - clean_paths = " ".join(f"{target_dir}/{item}" for item in sub_items) - clean_cmd = adb_cmd + ["shell", f"rm -rf {clean_paths}"] - logger.info(f"+ {' '.join(clean_cmd)}") - subprocess.run(clean_cmd) + if args.hex_only: + logger.info("\nPushing Hexagon libraries to Android device via ADB...") + target_lib_dir = f"{target_dir}/lib" + for lib in hex_libs: + push_cmd = adb_cmd + ["push", lib, f"{target_lib_dir}/"] + logger.info(f"+ {' '.join(push_cmd)}") + res = subprocess.run(push_cmd) + if res.returncode != 0: + logger.error("ADB push failed.") + sys.exit(res.returncode) + logger.info("Hexagon libraries deployed successfully!") + else: + # Clean stale package files on device + if sub_items: + clean_paths = " ".join(f"{target_dir}/{item}" for item in sub_items) + clean_cmd = adb_cmd + ["shell", f"rm -rf {clean_paths}"] + logger.info(f"+ {' '.join(clean_cmd)}") + subprocess.run(clean_cmd) - # Android destination directory is target_dir - push_cmd = adb_cmd + ["push", os.path.join(src_path, "."), target_dir] - logger.info(f"+ {' '.join(push_cmd)}") - res = subprocess.run(push_cmd) - if res.returncode != 0: - logger.error("ADB push failed.") - sys.exit(res.returncode) + # Android destination directory is target_dir + push_cmd = adb_cmd + ["push", os.path.join(src_path, "."), target_dir] + logger.info(f"+ {' '.join(push_cmd)}") + res = subprocess.run(push_cmd) + if res.returncode != 0: + logger.error("ADB push failed.") + sys.exit(res.returncode) - chmod_cmd = adb_cmd + ["shell", f"chmod -R 755 {target_dir}/bin 2>/dev/null || true"] - logger.info(f"+ {' '.join(chmod_cmd)}") - subprocess.run(chmod_cmd) - logger.info("ADB push completed successfully!") + chmod_cmd = adb_cmd + ["shell", f"chmod -R 755 {target_dir}/bin 2>/dev/null || true"] + logger.info(f"+ {' '.join(chmod_cmd)}") + subprocess.run(chmod_cmd) + logger.info("ADB push completed successfully!") elif target_type == "linux": ssh_host = target_val if not ssh_host: logger.error("Error: SSH host not specified in target (e.g. use linux:user@host, lnx:user@host, or ubuntu:user@host). Cannot deploy.") sys.exit(1) - logger.info(f"\nDeploying built artifacts to Linux device {ssh_host} via SSH/SCP...") - # Clean stale package files on remote host - if sub_items: - clean_paths = " ".join(f"{target_dir}/{item}" for item in sub_items) - clean_cmd = ["ssh", ssh_host, f"rm -rf {clean_paths}"] - logger.info(f"+ {' '.join(clean_cmd)}") - subprocess.run(clean_cmd) + if args.hex_only: + logger.info(f"\nDeploying Hexagon libraries to Linux device {ssh_host} via SCP...") + target_lib_dir = f"{target_dir}/lib" + deploy_cmd = ["scp"] + hex_libs + [f"{ssh_host}:{target_lib_dir}/"] + logger.info(f"+ {' '.join(deploy_cmd)}") + res = subprocess.run(deploy_cmd) + if res.returncode != 0: + logger.error("SCP deploy failed.") + sys.exit(res.returncode) + logger.info("Hexagon libraries deployed successfully!") + else: + # Clean stale package files on remote host + if sub_items: + clean_paths = " ".join(f"{target_dir}/{item}" for item in sub_items) + clean_cmd = ["ssh", ssh_host, f"rm -rf {clean_paths}"] + logger.info(f"+ {' '.join(clean_cmd)}") + subprocess.run(clean_cmd) - # Deploy to target_dir - deploy_cmd = ["scp", "-r", os.path.join(src_path, "."), f"{ssh_host}:{target_dir}"] - logger.info(f"+ {' '.join(deploy_cmd)}") - res = subprocess.run(deploy_cmd) - if res.returncode != 0: - logger.error("SSH/SCP deploy failed.") - sys.exit(res.returncode) + # Deploy to target_dir + deploy_cmd = ["scp", "-r", os.path.join(src_path, "."), f"{ssh_host}:{target_dir}"] + logger.info(f"+ {' '.join(deploy_cmd)}") + res = subprocess.run(deploy_cmd) + if res.returncode != 0: + logger.error("SSH/SCP deploy failed.") + sys.exit(res.returncode) - chmod_cmd = ["ssh", ssh_host, f"chmod -R 755 {target_dir}/bin 2>/dev/null || true"] - logger.info(f"+ {' '.join(chmod_cmd)}") - subprocess.run(chmod_cmd) - logger.info("SSH/SCP deploy completed successfully!") + chmod_cmd = ["ssh", ssh_host, f"chmod -R 755 {target_dir}/bin 2>/dev/null || true"] + logger.info(f"+ {' '.join(chmod_cmd)}") + subprocess.run(chmod_cmd) + logger.info("SSH/SCP deploy completed successfully!") elif target_type == "windows": logger.info("\nPush for Windows on Snapdragon (windows) target is currently a stub.")