hexagon: optimize CPY/CONCAT — round 3, short-rows edition (#30341)

* hexagon: move the converting CPY in blocks of rows per DMA descriptor so short rows are not DMA-latency bound

* hex-cpy: cleanup cpy cursor handling

* hex-cpy-concat: introduce threaded dma/hvx gather-row helper

Co-authored-by: Ruslan Karymov <67752306+karusrus@users.noreply.github.com>

* hex-cpy: remove leftover rejections for extended buffers

* hex-cpy: relax the gate for fast-path

* hex-cpy: remove scalar divs and vtcm access and use explicit gather_sync

* hex-cpy: add proper support for mdev work splitting to fastpath

* hex-cpy: optimize dense gather-row tasks

* hex-cpy: unroll gather-rows loops

* hex-cpy: add fastpath for f16

* hex-cpy-concat: support for colapsing 3/4D dims

* hex-cpy: double-buffer support

* hex-cpy: factor out gather-rows into more of an op by itself

* hex-cpy-concat: dedup kernel-params

* hex-cpy-concat: step-down nr to fit instead of hard fail

* hex-gather: replace div with a mask

* hex-gather-rows: cleanup dma pipeline and remove the stalls

* hex-gather: specialize compute funcs

* hex-gather: reduce scalar reg.spills and func calls

* hex-gather: do not recompute rows-per-thread on the npu

* hex-build: add support for building/installing just hexagon

* hex-cpy-concat: remove redundant barrier in concat and move cpy barrier under fence

* hex-gather: annotations for dma-out

* hex-mdev: make all key vtcm_acquire sync state atomic (fixes random hangs in mdev setups)

---------

Co-authored-by: Yiwei Shao <yiwei@aizip.ai>
Co-authored-by: Ruslan Karymov <67752306+karusrus@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-10-12 07:39:55 -07:00
1 parent 0f701c8829
commit 8bb3cf0d21
10 files changed
+818 -124

No files matched your search

+61 -6
View File
@@ -6593,6 +6593,37 @@ static bool ggml_hexagon_precompute_concat_params(
}
}
const uint32_t max_gather_ne0 = 128 / type_size;
const uint32_t total_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]);
const bool b_dense = (src1->nb[0] == type_size);
const bool gather_strides_ok =
src0->nb[1] >= (size_t) src0->ne[0] * type_size &&
(b_dense ? src1->nb[1] >= (size_t) src1->ne[0] * type_size
: src1->nb[0] >= (size_t) src1->ne[1] * type_size);
if (dim == 0 &&
(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16) &&
dst->ne[0] <= max_gather_ne0 &&
dst->nb[0] == type_size && dst->nb[1] == (size_t) dst->ne[0] * type_size &&
src0->nb[0] == type_size &&
(src1->nb[0] == type_size || src1->nb[1] == type_size) &&
gather_strides_ok &&
htp_gather_rows_tensor_outer_contiguous(dst) &&
htp_gather_rows_tensor_outer_contiguous(src0) &&
htp_gather_rows_tensor_outer_contiguous(src1)) {
if (htp_gather_rows_solve_layout(&kparams->u.gather,
(uint32_t) src0->ne[0], (uint32_t) src0->nb[1],
(uint32_t) src1->ne[0], (uint32_t) src1->nb[1], b_dense,
type_size, total_rows,
sess->n_threads, sess->mdev.count, sess->vtcm_size)) {
kparams->kernel_type = HTP_CONCAT_KERNEL_GATHER_ROWS;
kparams->n_threads = (uint8_t) sess->n_threads;
kparams->vtcm_size = kparams->u.gather.slice * sess->n_threads;
return true;
}
}
const bool dma_strides_ok = (src0->nb[0] == type_size && src1->nb[0] == type_size && dst->nb[0] == type_size);
if (dma_strides_ok) {
@@ -6616,11 +6647,11 @@ static bool ggml_hexagon_precompute_concat_params(
return false;
}
kparams->kernel_type = HTP_CONCAT_KERNEL_TRANSPOSED;
kparams->n_threads = n_threads;
kparams->vtcm_size = layout.total_bytes;
kparams->spad0_size_per_thread = layout.src0_spad_size_per_thread;
kparams->spad1_size_per_thread = layout.src1_spad_size_per_thread;
kparams->kernel_type = HTP_CONCAT_KERNEL_TRANSPOSED;
kparams->n_threads = n_threads;
kparams->vtcm_size = layout.total_bytes;
kparams->u.transposed.spad0_size_per_thread = layout.src0_spad_size_per_thread;
kparams->u.transposed.spad1_size_per_thread = layout.src1_spad_size_per_thread;
return true;
}
@@ -6706,6 +6737,27 @@ static bool ggml_hexagon_precompute_cpy_params(
return true;
}
const uint32_t max_gather_ne0 = 128 / src_type_size;
const uint32_t total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);
if ((src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) &&
dst_is_contiguous && !src_is_contiguous &&
src0->nb[0] == src_type_size && src0->ne[0] <= max_gather_ne0 &&
src0->nb[1] >= (size_t) src0->ne[0] * src_type_size &&
total_rows > 1 &&
htp_gather_rows_tensor_outer_contiguous(src0)) {
if (htp_gather_rows_solve_layout(&kparams->u.gather,
(uint32_t) src0->ne[0], (uint32_t) src0->nb[1],
0, 0, false,
src_type_size, total_rows,
sess->n_threads, sess->mdev.count, sess->vtcm_size)) {
kparams->kernel_type = HTP_COPY_KERNEL_GATHER_ROWS;
kparams->n_threads = (uint8_t) sess->n_threads;
kparams->vtcm_size = kparams->u.gather.slice * sess->n_threads;
return true;
}
}
if (sameshape) {
kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_SAMETYPE;
kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);
@@ -6737,7 +6789,7 @@ static bool ggml_hexagon_precompute_cpy_params(
const uint32_t n_threads = sess->n_threads > 0 ? (uint32_t) sess->n_threads : 4;
struct htp_copy_convert_vtcm_layout layout;
htp_copy_convert_vtcm_layout_build(&layout, (uint32_t) src0->ne[0], src_type_size, dst_type_size, n_threads);
htp_copy_convert_vtcm_layout_build(&layout, (uint32_t) src0->ne[0], (uint32_t) src0->ne[1], src_type_size, dst_type_size, n_threads);
if (sess->vtcm_size > 0 && layout.total_bytes > sess->vtcm_size) {
return false;
@@ -6751,6 +6803,9 @@ static bool ggml_hexagon_precompute_cpy_params(
kparams->u.convert.dst_buf_size = layout.dst_buf_size;
kparams->u.convert.spad0_size_per_thread = layout.spad0_size_per_thread;
kparams->u.convert.spad1_size_per_thread = layout.spad1_size_per_thread;
kparams->u.convert.src0_row_stride = layout.src0_row_stride;
kparams->u.convert.dst_row_stride = layout.dst_row_stride;
kparams->u.convert.blk_rows = layout.blk_rows;
kparams->u.convert.div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]);
kparams->u.convert.div_ne02_ne01 = init_fastdiv_values((uint32_t) (src0->ne[2] * src0->ne[1]));
+1
View File
@@ -33,6 +33,7 @@ add_library(${HTP_LIB} SHARED
set-rows-ops.c
get-rows-ops.c
cpy-ops.c
gather-rows.c
repeat-ops.c
ssm-conv.c
cumsum-ops.c
+13 -9
View File
@@ -13,6 +13,7 @@
#include "htp-vtcm.h"
#include "hvx-utils.h"
#include "hvx_hexagon_protos.h"
#include "gather-rows.h"
#include <string.h>
@@ -302,9 +303,9 @@ static int concat_transposed(struct htp_ops_context * octx, const struct htp_con
struct htp_concat_context cctx;
cctx.octx = octx;
cctx.spad0_base = vtcm_base;
cctx.spad1_base = vtcm_base + n_threads * kparams->spad0_size_per_thread;
cctx.spad0_size_per_thread = kparams->spad0_size_per_thread;
cctx.spad1_size_per_thread = kparams->spad1_size_per_thread;
cctx.spad1_base = vtcm_base + n_threads * kparams->u.transposed.spad0_size_per_thread;
cctx.spad0_size_per_thread = kparams->u.transposed.spad0_size_per_thread;
cctx.spad1_size_per_thread = kparams->u.transposed.spad1_size_per_thread;
cctx.row_start = row_start;
cctx.nrows = nrows;
cctx.nplanes = dst->ne[2] * dst->ne[3];
@@ -318,10 +319,13 @@ static int concat_transposed(struct htp_ops_context * octx, const struct htp_con
int op_concat(struct htp_ops_context * octx) {
const struct htp_concat_kernel_params * kparams = (const struct htp_concat_kernel_params *) octx->kernel_params;
const struct htp_tensor * dst = octx->dst;
const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2;
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * src1 = octx->src[1];
const struct htp_tensor * dst = octx->dst;
int status = HTP_STATUS_OK;
const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2;
switch (kparams->kernel_type) {
case HTP_CONCAT_KERNEL_REGULAR:
status = concat_regular(octx, kparams->dim, type_size);
@@ -331,6 +335,10 @@ int op_concat(struct htp_ops_context * octx) {
status = concat_transposed(octx, kparams, type_size);
break;
case HTP_CONCAT_KERNEL_GATHER_ROWS:
status = htp_gather_rows(octx, src0, src1, &kparams->u.gather, kparams->n_threads);
break;
default:
status = HTP_STATUS_NO_SUPPORT;
break;
@@ -338,9 +346,5 @@ int op_concat(struct htp_ops_context * octx) {
htp_ops_context_set_status(octx, status);
if (octx->ctx->mdev.count > 1) {
htp_mdev_group_barrier(octx);
}
return octx->status;
}
+12 -2
View File
@@ -2,12 +2,19 @@
#define HTP_CONCAT_OPS_H
#include "hex-common.h"
#include "gather-rows.h"
#include <stdint.h>
enum htp_concat_kernel_type {
HTP_CONCAT_KERNEL_UNSUPPORTED = 0,
HTP_CONCAT_KERNEL_REGULAR = 1,
HTP_CONCAT_KERNEL_TRANSPOSED = 2,
HTP_CONCAT_KERNEL_GATHER_ROWS = 3,
};
struct htp_concat_transposed_params {
uint32_t spad0_size_per_thread;
uint32_t spad1_size_per_thread;
};
struct htp_concat_kernel_params {
@@ -17,8 +24,11 @@ struct htp_concat_kernel_params {
uint8_t pad;
uint32_t vtcm_size;
uint32_t spad0_size_per_thread;
uint32_t spad1_size_per_thread;
union {
struct htp_concat_transposed_params transposed;
struct htp_gather_rows_params gather;
} u;
};
#if defined(__cplusplus)
+55 -57
View File
@@ -18,6 +18,7 @@
#include "htp-ops.h"
#include "htp-tensor.h"
#include "hvx-utils.h"
#include "gather-rows.h"
struct htp_copy_context {
struct htp_ops_context * octx;
@@ -141,6 +142,25 @@ DEFINE_CPY_RESHAPE(f32, float, 4)
DEFINE_CPY_RESHAPE(f16, __fp16, 2)
DEFINE_CPY_RESHAPE(i32, int32_t, 4)
struct cpy_row_cursor {
uint32_t r;
uint32_t i01;
uint32_t i02;
uint32_t i03;
};
static inline void cpy_cursor_advance(struct cpy_row_cursor * c, uint32_t n, uint32_t ne01, uint32_t ne02) {
c->r += n;
c->i01 += n;
if (c->i01 == ne01) {
c->i01 = 0;
if (++c->i02 == ne02) {
c->i02 = 0;
c->i03++;
}
}
}
#define DEFINE_CPY_CONVERT_SAMESHAPE(NAME, CONV_FUNC) \
static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \
struct htp_copy_context * ct = (struct htp_copy_context *) data; \
@@ -151,7 +171,6 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vo
const uint32_t ir0 = ct->row_start + dr * ith; \
const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); \
if (ir0 >= ir1) return; \
const uint32_t nrows_thread = ir1 - ir0; \
\
dma_queue * dma_q = octx->ctx->dma[ith]; \
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
@@ -159,6 +178,9 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vo
const struct htp_copy_convert_params * cvt = &ct->kparams->u.convert; \
const uint32_t src0_buf_size = cvt->src0_buf_size; \
const uint32_t dst_buf_size = cvt->dst_buf_size; \
const uint32_t src0_row_stride = cvt->src0_row_stride; \
const uint32_t dst_row_stride = cvt->dst_row_stride; \
const uint32_t blk_rows = cvt->blk_rows; \
uint8_t * vtcm_src0_base = ct->vtcm_src0 + ith * cvt->spad0_size_per_thread; \
uint8_t * vtcm_dst_base = ct->vtcm_dst + ith * cvt->spad1_size_per_thread; \
const uint32_t src0_row_size = ne00 * ct->kparams->src0_type_size; \
@@ -170,62 +192,43 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vo
uint32_t i02 = fastdiv(rem, &cvt->div_ne01); \
uint32_t i01 = rem - i02 * ne01; \
\
uint32_t f_i01 = i01, f_i02 = i02, f_i03 = i03; \
dma_addr_t f_src0_addr = src0->data + f_i01*nb01 + f_i02*nb02 + f_i03*nb03; \
struct cpy_row_cursor f = { ir0, i01, i02, i03 }; \
struct cpy_row_cursor c = f; \
\
uint32_t c_i01 = i01, c_i02 = i02, c_i03 = i03; \
dma_addr_t c_dst_addr = dst->data + c_i01*nb1 + c_i02*nb2 + c_i03*nb3; \
\
for (uint32_t r = 0; r < nrows_thread && r < 2; ++r) { \
uint8_t * src_spad = vtcm_src0_base + r * src0_buf_size; \
uint8_t * dst_spad = vtcm_dst_base + r * dst_buf_size; \
for (uint32_t b = 0; b < 2 && f.r < ir1; ++b) { \
uint8_t * src_spad = vtcm_src0_base + b * src0_buf_size; \
uint8_t * dst_spad = vtcm_dst_base + b * dst_buf_size; \
const uint32_t nr = MIN(blk_rows, MIN(ne01 - f.i01, ir1 - f.r)); \
dma_addr_t f_src0_addr = src0->data + f.i01*nb01 + f.i02*nb02 + f.i03*nb03; \
dma_queue_push(dma_q, dma_make_data(dst->data, dst_spad), \
dst_row_size, dst_buf_size, dst_row_size, 0); \
dma_queue_push(dma_q, dma_make_data(src_spad, f_src0_addr), \
src0_buf_size, src0_row_size, src0_row_size, 1); \
f_src0_addr += nb01; \
if (++f_i01 == ne01) { \
f_i01 = 0; \
if (++f_i02 == ne02) { \
f_i02 = 0; \
f_i03++; \
} \
f_src0_addr = src0->data + f_i02*nb02 + f_i03*nb03; \
} \
src0_row_stride, nb01, src0_row_size, nr); \
cpy_cursor_advance(&f, nr, ne01, ne02); \
} \
\
for (uint32_t r = 0; r < nrows_thread; ++r) { \
for (uint32_t blk = 0; c.r < ir1; ++blk) { \
const uint32_t nr = MIN(blk_rows, MIN(ne01 - c.i01, ir1 - c.r)); \
uint8_t * dst_spad = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).src; \
uint8_t * src_spad = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).dst; \
\
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \
CONV_FUNC(dst_spad, src_spad, ne00); \
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \
\
dma_queue_push(dma_q, dma_make_data(c_dst_addr, dst_spad), \
dst_row_size, dst_buf_size, dst_row_size, 1); \
c_dst_addr += nb1; \
if (++c_i01 == ne01) { \
c_i01 = 0; \
if (++c_i02 == ne02) { \
c_i02 = 0; \
c_i03++; \
} \
c_dst_addr = dst->data + c_i02*nb2 + c_i03*nb3; \
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) blk); \
for (uint32_t r = 0; r < nr; ++r) { \
CONV_FUNC(dst_spad + r * dst_row_stride, src_spad + r * src0_row_stride, ne00); \
} \
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) blk); \
\
if (r + 2 < nrows_thread) { \
dma_addr_t c_dst_addr = dst->data + c.i01*nb1 + c.i02*nb2 + c.i03*nb3; \
dma_queue_push(dma_q, dma_make_data(c_dst_addr, dst_spad), \
nb1, dst_row_stride, dst_row_size, nr); \
cpy_cursor_advance(&c, nr, ne01, ne02); \
\
if (f.r < ir1) { \
const uint32_t fnr = MIN(blk_rows, MIN(ne01 - f.i01, ir1 - f.r)); \
dma_addr_t f_src0_addr = src0->data + f.i01*nb01 + f.i02*nb02 + f.i03*nb03; \
dma_queue_push(dma_q, dma_make_data(src_spad, f_src0_addr), \
src0_buf_size, src0_row_size, src0_row_size, 1); \
f_src0_addr += nb01; \
if (++f_i01 == ne01) { \
f_i01 = 0; \
if (++f_i02 == ne02) { \
f_i02 = 0; \
f_i03++; \
} \
f_src0_addr = src0->data + f_i02*nb02 + f_i03*nb03; \
} \
src0_row_stride, nb01, src0_row_size, fnr); \
cpy_cursor_advance(&f, fnr, ne01, ne02); \
} \
} \
dma_queue_flush(dma_q); \
@@ -337,10 +340,6 @@ static int cpy_sameshape_convert(struct htp_ops_context * octx, const struct htp
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) {
return HTP_STATUS_NO_SUPPORT;
}
if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -398,10 +397,6 @@ static int cpy_reshape(struct htp_ops_context * octx, const struct htp_copy_kern
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) {
return HTP_STATUS_NO_SUPPORT;
}
if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
}
@@ -461,6 +456,9 @@ int op_cpy(struct htp_ops_context * octx) {
case HTP_COPY_KERNEL_RESHAPE:
status = cpy_reshape(octx, kparams);
break;
case HTP_COPY_KERNEL_GATHER_ROWS:
status = htp_gather_rows(octx, octx->src[0], NULL, &kparams->u.gather, kparams->n_threads);
break;
default:
status = HTP_STATUS_NO_SUPPORT;
break;
@@ -468,11 +466,11 @@ int op_cpy(struct htp_ops_context * octx) {
htp_ops_context_set_status(octx, status);
if (octx->ctx->mdev.count > 1) {
htp_mdev_group_barrier(octx);
}
if (octx->op == HTP_OP_CPY_FENCE) {
if (octx->ctx->mdev.count > 1) {
htp_mdev_group_barrier(octx);
}
if (octx->ctx->mdev.idx == 0) {
const struct htp_tensor * sync = octx->src[1];
const uint32_t seq = (uint32_t) octx->op_params[0];
+31 -2
View File
@@ -3,6 +3,7 @@
#include "hex-common.h"
#include "hex-fastdiv.h"
#include "gather-rows.h"
#include <stdint.h>
enum htp_copy_kernel_type {
@@ -12,6 +13,7 @@ enum htp_copy_kernel_type {
HTP_COPY_KERNEL_SAMESHAPE_CONVERT = 3,
HTP_COPY_KERNEL_RESHAPE = 4,
HTP_COPY_KERNEL_SCALAR = 5,
HTP_COPY_KERNEL_GATHER_ROWS = 6,
};
struct htp_copy_convert_params {
@@ -19,6 +21,9 @@ struct htp_copy_convert_params {
uint32_t dst_buf_size;
uint32_t spad0_size_per_thread;
uint32_t spad1_size_per_thread;
uint32_t src0_row_stride; // row pitch in the VTCM buffers
uint32_t dst_row_stride;
uint32_t blk_rows; // rows moved per DMA descriptor
struct fastdiv_values div_ne01;
struct fastdiv_values div_ne02_ne01;
};
@@ -45,10 +50,19 @@ struct htp_copy_kernel_params {
union {
struct htp_copy_convert_params convert;
struct htp_copy_reshape_params reshape;
struct htp_gather_rows_params gather;
} u;
};
// Size of one VTCM staging buffer: big enough that a DMA descriptor moves many short
// rows at once. The copy kernels keep four per thread (two in flight each way), about
// 1 MB of VTCM with 8 threads.
#define HTP_COPY_VTCM_BLK_BYTES (32 * 1024)
struct htp_copy_convert_vtcm_layout {
uint32_t src0_row_stride;
uint32_t dst_row_stride;
uint32_t blk_rows;
uint32_t src0_buf_size;
uint32_t dst_buf_size;
uint32_t spad0_size_per_thread;
@@ -56,15 +70,30 @@ struct htp_copy_convert_vtcm_layout {
uint32_t total_bytes;
};
// ne01 bounds a block: rows of one block share the dim-1 stride.
static inline void htp_copy_convert_vtcm_layout_build(
struct htp_copy_convert_vtcm_layout * layout,
uint32_t ne00,
uint32_t ne01,
uint32_t src_type_size,
uint32_t dst_type_size,
uint32_t n_threads) {
layout->src0_buf_size = hex_round_up(ne00 * src_type_size, 256);
layout->dst_buf_size = hex_round_up(ne00 * dst_type_size, 256);
layout->src0_row_stride = hex_round_up(ne00 * src_type_size, 256);
layout->dst_row_stride = hex_round_up(ne00 * dst_type_size, 256);
const uint32_t max_row = layout->src0_row_stride > layout->dst_row_stride ? layout->src0_row_stride : layout->dst_row_stride;
uint32_t blk_rows = HTP_COPY_VTCM_BLK_BYTES / max_row;
if (blk_rows > ne01) {
blk_rows = ne01;
}
if (blk_rows == 0) {
blk_rows = 1;
}
layout->blk_rows = blk_rows;
layout->src0_buf_size = blk_rows * layout->src0_row_stride;
layout->dst_buf_size = blk_rows * layout->dst_row_stride;
layout->spad0_size_per_thread = 2 * layout->src0_buf_size;
layout->spad1_size_per_thread = 2 * layout->dst_buf_size;
layout->total_bytes = n_threads * (layout->spad0_size_per_thread + layout->spad1_size_per_thread);
+387
View File
@@ -0,0 +1,387 @@
#include "gather-rows.h"
#include <assert.h>
#include "hex-common.h"
#include "hex-profile.h"
#include "hvx-utils.h"
#include "dma-queue.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "work-queue.h"
typedef void (*gather_rows_compute_fn_t)(
uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,
const struct htp_gather_rows_params * restrict kparams,
const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur);
struct gather_rows_task {
struct htp_ops_context * octx;
const struct htp_tensor * a;
const struct htp_tensor * b;
dma_addr_t dst;
uint32_t r_start;
uint32_t rows;
uint32_t rows_per_thread;
const struct htp_gather_rows_params * kparams;
gather_rows_compute_fn_t compute;
};
static inline __attribute__((always_inline)) void gather_rows_dma_in(
dma_queue * q, const struct gather_rows_task * task,
uint8_t * region, uint32_t r, uint32_t nr, uint32_t elem_size) {
const struct htp_tensor * a = task->a;
const struct htp_tensor * b = task->b;
const struct htp_gather_rows_params * kparams = task->kparams;
const size_t a_size = (nr - 1) * a->nb[1] + kparams->na * elem_size;
dma_queue_push(q, dma_make_data(region, a->data + r * a->nb[1]), a_size, a_size, a_size, 1);
if (b) {
if (kparams->b_dense) {
const size_t b_size = (nr - 1) * b->nb[1] + kparams->nb * elem_size;
dma_queue_push(q, dma_make_data(region + kparams->v1, b->data + r * b->nb[1]), b_size, b_size, b_size, 1);
} else {
const size_t b_size = (nr - 1) * b->nb[1] + elem_size;
for (uint32_t k = 0; k < kparams->nb; k++) {
dma_queue_push(q, dma_make_data(region + kparams->v1 + k * kparams->span1,
b->data + k * b->nb[0] + r * b->nb[1]), b_size, b_size, b_size, 1);
}
}
}
}
static inline void gather_rows_dma_out(dma_queue * q, const struct gather_rows_task * task,
uint8_t * out, uint32_t r, uint32_t nr, uint32_t elem_size) {
const struct htp_gather_rows_params * kparams = task->kparams;
const size_t out_size = nr * kparams->ne * elem_size;
dma_queue_push(q, dma_make_data(task->dst + r * kparams->ne * elem_size, out), out_size, out_size, out_size, 1);
}
static __attribute__((noinline)) void gather_rows_init_tables_f32(
HVX_Vector * tab, HVX_Vector * inc,
const struct htp_gather_rows_params * kparams,
const struct htp_tensor * a, const struct htp_tensor * b) {
const int32_t s_a = kparams->rg * a->nb[1];
const int32_t s_b = b ? kparams->rg * b->nb[1] : 0;
const uint32_t b_nb1 = b ? b->nb[1] : 0;
uint32_t row = 0;
uint32_t col = 0;
int32_t t_buf[32] __attribute__((aligned(128)));
int32_t s_buf[32] __attribute__((aligned(128)));
for (uint32_t k = 0; k < kparams->vg; k++) {
for (uint32_t lane = 0; lane < 32; lane++) {
if (col < kparams->na) {
t_buf[lane] = row * a->nb[1] + col * 4;
s_buf[lane] = s_a;
} else {
t_buf[lane] = kparams->v1 + (col - kparams->na) * kparams->span1 + row * b_nb1;
s_buf[lane] = s_b;
}
if (++col == kparams->ne) {
col = 0;
row++;
}
}
tab[k] = *(const HVX_Vector *) t_buf;
inc[k] = *(const HVX_Vector *) s_buf;
}
}
static __attribute__((noinline)) void gather_rows_init_tables_f16(
HVX_Vector * tab, HVX_Vector * inc,
const struct htp_gather_rows_params * kparams,
const struct htp_tensor * a, const struct htp_tensor * b) {
const int16_t s_a = (int16_t) (kparams->rg * a->nb[1]);
const int16_t s_b = (int16_t) (b ? kparams->rg * b->nb[1] : 0);
const uint32_t b_nb1 = b ? b->nb[1] : 0;
uint32_t row = 0;
uint32_t col = 0;
int16_t t_buf[64] __attribute__((aligned(128)));
int16_t s_buf[64] __attribute__((aligned(128)));
for (uint32_t k = 0; k < kparams->vg; k++) {
for (uint32_t lane = 0; lane < 64; lane++) {
if (col < kparams->na) {
t_buf[lane] = (int16_t) (row * a->nb[1] + col * 2);
s_buf[lane] = s_a;
} else {
t_buf[lane] = (int16_t) (kparams->v1 + (col - kparams->na) * kparams->span1 + row * b_nb1);
s_buf[lane] = s_b;
}
if (++col == kparams->ne) {
col = 0;
row++;
}
}
tab[k] = *(const HVX_Vector *) t_buf;
inc[k] = *(const HVX_Vector *) s_buf;
}
}
static void gather_rows_compute_f32_vg1(
uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,
const struct htp_gather_rows_params * restrict kparams,
const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) {
(void) cur;
const uint32_t region_size = kparams->region_size;
HVX_Vector vcur = tab[0];
const HVX_Vector vinc = inc[0];
#pragma unroll(4)
for (uint32_t v = 0; v < nvec; v++) {
Q6_vgather_ARMVw((HVX_Vector *) (out + v * 128), (size_t) region, region_size, vcur);
vcur = Q6_Vw_vadd_VwVw(vcur, vinc);
}
}
static void gather_rows_compute_f32_vgn(
uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,
const struct htp_gather_rows_params * restrict kparams,
const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) {
const uint32_t vg = kparams->vg;
const uint32_t region_size = kparams->region_size;
for (uint32_t k = 0; k < vg; k++) {
cur[k] = tab[k];
}
uint32_t k = 0;
#pragma unroll(4)
for (uint32_t v = 0; v < nvec; v++) {
Q6_vgather_ARMVw((HVX_Vector *) (out + v * 128), (size_t) region, region_size, cur[k]);
cur[k] = Q6_Vw_vadd_VwVw(cur[k], inc[k]);
if (++k == vg) {
k = 0;
}
}
}
static void gather_rows_compute_f16_vg1(
uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,
const struct htp_gather_rows_params * restrict kparams,
const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) {
(void) cur;
const uint32_t region_size = kparams->region_size;
HVX_Vector vcur = tab[0];
const HVX_Vector vinc = inc[0];
#pragma unroll(4)
for (uint32_t v = 0; v < nvec; v++) {
Q6_vgather_ARMVh((HVX_Vector *) (out + v * 128), (size_t) region, region_size, vcur);
vcur = Q6_Vh_vadd_VhVh(vcur, vinc);
}
}
static void gather_rows_compute_f16_vgn(
uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,
const struct htp_gather_rows_params * restrict kparams,
const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) {
const uint32_t vg = kparams->vg;
const uint32_t region_size = kparams->region_size;
for (uint32_t k = 0; k < vg; k++) {
cur[k] = tab[k];
}
uint32_t k = 0;
#pragma unroll(4)
for (uint32_t v = 0; v < nvec; v++) {
Q6_vgather_ARMVh((HVX_Vector *) (out + v * 128), (size_t) region, region_size, cur[k]);
cur[k] = Q6_Vh_vadd_VhVh(cur[k], inc[k]);
if (++k == vg) {
k = 0;
}
}
}
static void gather_rows_thread_f32(unsigned int nth, unsigned int ith, void * data) {
(void) nth;
const struct gather_rows_task * task = (const struct gather_rows_task *) data;
const struct htp_gather_rows_params * kparams = task->kparams;
const struct htp_tensor * a = task->a;
const struct htp_tensor * b = task->b;
const uint32_t r_offset = MIN(ith * task->rows_per_thread, task->rows);
const uint32_t r_beg = task->r_start + r_offset;
const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_thread, task->rows);
if (r_beg >= r_end) {
return;
}
struct htp_thread_trace * tr = &task->octx->ctx->trace[ith];
uint8_t * base = task->octx->ctx->vtcm_base + ith * kparams->slice;
HVX_Vector * tab = (HVX_Vector *) base;
HVX_Vector * inc = tab + kparams->vg;
HVX_Vector * cur = inc + kparams->vg;
uint8_t * region_base = (uint8_t *) (cur + kparams->vg);
uint8_t * region[2] = { region_base, region_base + kparams->buf_stride };
uint8_t * out[2] = { region[0] + kparams->out_off, region[1] + kparams->out_off };
htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);
gather_rows_init_tables_f32(tab, inc, kparams, a, b);
htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);
dma_queue * q = task->octx->ctx->dma[ith];
const uint32_t n_in = 1 + (task->b ? (kparams->b_dense ? 1 : kparams->nb) : 0);
const uint32_t nr_max = kparams->nr_max;
const uint32_t nr0 = MIN(nr_max, r_end - r_beg);
dma_queue_push(q, dma_make_data(task->dst, out[0]), 0, 0, 0, 0); // dummy out
gather_rows_dma_in(q, task, region[0], r_beg, nr0, 4);
const uint32_t r1 = r_beg + nr_max;
if (r1 < r_end) {
const uint32_t nr1 = MIN(nr_max, r_end - r1);
dma_queue_push(q, dma_make_data(task->dst, out[1]), 0, 0, 0, 0); // dummy out
gather_rows_dma_in(q, task, region[1], r1, nr1, 4);
}
uint32_t buf = 0;
for (uint32_t r = r_beg; r < r_end; r += nr_max) {
const uint32_t nr = MIN(nr_max, r_end - r);
const uint32_t next_buf = buf ^ 1;
dma_queue_pop(q); // complete out
for (uint32_t i = 0; i < n_in; i++) {
dma_queue_pop(q);
}
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r);
const uint32_t nvec = (nr * kparams->ne + 31) >> 5;
task->compute(out[buf], region[buf], nvec, kparams, tab, inc, cur);
hvx_gather_sync(out[buf]);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r);
gather_rows_dma_out(q, task, out[buf], r, nr, 4);
const uint32_t r_prefetch = r + 2 * nr_max;
if (r_prefetch < r_end) {
const uint32_t nr_prefetch = MIN(nr_max, r_end - r_prefetch);
gather_rows_dma_in(q, task, region[buf], r_prefetch, nr_prefetch, 4);
}
buf = next_buf;
}
dma_queue_flush(q);
}
static void gather_rows_thread_f16(unsigned int nth, unsigned int ith, void * data) {
(void) nth;
const struct gather_rows_task * task = (const struct gather_rows_task *) data;
const struct htp_gather_rows_params * kparams = task->kparams;
const struct htp_tensor * a = task->a;
const struct htp_tensor * b = task->b;
const uint32_t r_offset = MIN(ith * task->rows_per_thread, task->rows);
const uint32_t r_beg = task->r_start + r_offset;
const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_thread, task->rows);
if (r_beg >= r_end) {
return;
}
struct htp_thread_trace * tr = &task->octx->ctx->trace[ith];
uint8_t * base = task->octx->ctx->vtcm_base + ith * kparams->slice;
HVX_Vector * tab = (HVX_Vector *) base;
HVX_Vector * inc = tab + kparams->vg;
HVX_Vector * cur = inc + kparams->vg;
uint8_t * region_base = (uint8_t *) (cur + kparams->vg);
uint8_t * region[2] = { region_base, region_base + kparams->buf_stride };
uint8_t * out[2] = { region[0] + kparams->out_off, region[1] + kparams->out_off };
htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);
gather_rows_init_tables_f16(tab, inc, kparams, a, b);
htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);
dma_queue * q = task->octx->ctx->dma[ith];
const uint32_t n_in = 1 + (task->b ? (kparams->b_dense ? 1 : kparams->nb) : 0);
const uint32_t nr_max = kparams->nr_max;
const uint32_t nr0 = MIN(nr_max, r_end - r_beg);
dma_queue_push(q, dma_make_data(task->dst, out[0]), 0, 0, 0, 0); // dummy out
gather_rows_dma_in(q, task, region[0], r_beg, nr0, 2);
const uint32_t r1 = r_beg + nr_max;
if (r1 < r_end) {
const uint32_t nr1 = MIN(nr_max, r_end - r1);
dma_queue_push(q, dma_make_data(task->dst, out[1]), 0, 0, 0, 0); // dummy out
gather_rows_dma_in(q, task, region[1], r1, nr1, 2);
}
uint32_t buf = 0;
for (uint32_t r = r_beg; r < r_end; r += nr_max) {
const uint32_t nr = MIN(nr_max, r_end - r);
const uint32_t next_buf = buf ^ 1;
dma_queue_pop(q); // complete out
for (uint32_t i = 0; i < n_in; i++) {
dma_queue_pop(q);
}
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r);
const uint32_t nvec = (nr * kparams->ne + 63) >> 6;
task->compute(out[buf], region[buf], nvec, kparams, tab, inc, cur);
hvx_gather_sync(out[buf]);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r);
gather_rows_dma_out(q, task, out[buf], r, nr, 2);
const uint32_t r_prefetch = r + 2 * nr_max;
if (r_prefetch < r_end) {
const uint32_t nr_prefetch = MIN(nr_max, r_end - r_prefetch);
gather_rows_dma_in(q, task, region[buf], r_prefetch, nr_prefetch, 2);
}
buf = next_buf;
}
dma_queue_flush(q);
}
int htp_gather_rows(
struct htp_ops_context * octx,
const struct htp_tensor * a,
const struct htp_tensor * b,
const struct htp_gather_rows_params * kparams,
uint32_t n_threads) {
if (!htp_ops_context_set_n_threads(octx, n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
}
assert(kparams->slice * octx->n_threads <= octx->ctx->vtcm_size);
const uint32_t total_rows = kparams->total_rows;
uint32_t r_start = 0;
uint32_t r_count = total_rows;
if (octx->ctx->mdev.count > 1) {
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(
total_rows, kparams->rg, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
r_start = range.start;
r_count = range.count;
}
if (r_count == 0) {
return HTP_STATUS_OK;
}
const uint32_t elem_size = (a->type == HTP_TYPE_F16) ? 2 : 4;
struct gather_rows_task task;
task.octx = octx;
task.a = a;
task.b = b;
task.dst = octx->dst->data;
task.r_start = r_start;
task.rows = r_count;
task.kparams = kparams;
task.rows_per_thread = kparams->rows_per_thread;
if (elem_size == 2) {
task.compute = (kparams->vg == 1) ? gather_rows_compute_f16_vg1 : gather_rows_compute_f16_vgn;
} else {
task.compute = (kparams->vg == 1) ? gather_rows_compute_f32_vg1 : gather_rows_compute_f32_vgn;
}
work_queue_func_t worker_func = (elem_size == 2) ? gather_rows_thread_f16 : gather_rows_thread_f32;
work_queue_run(octx->ctx->work_queue, worker_func, &task, octx->n_threads);
return HTP_STATUS_OK;
}
+160
View File
@@ -0,0 +1,160 @@
#ifndef HTP_GATHER_ROWS_H
#define HTP_GATHER_ROWS_H
#include "hex-common.h"
#include <stdint.h>
#include <stdbool.h>
#ifndef GGML_MAX_DIMS
#define GGML_MAX_DIMS 4
#endif
struct htp_gather_rows_params {
uint32_t na;
uint32_t nb;
uint32_t ne;
uint32_t rg;
uint32_t vg;
uint32_t nr_max;
uint32_t v1;
uint32_t span1;
uint32_t region_size;
uint32_t out_off;
uint32_t buf_stride;
uint32_t slice;
uint32_t total_rows;
uint32_t rows_per_thread;
uint8_t b_dense;
uint8_t pad[3];
};
static inline bool htp_gather_rows_solve_layout(
struct htp_gather_rows_params * kparams,
uint32_t na, uint32_t a_nb1,
uint32_t nb, uint32_t b_nb1, bool b_dense,
uint32_t elem_size,
uint32_t total_rows,
uint32_t n_threads,
uint32_t mdev_count,
uint32_t vtcm_size) {
const uint32_t max_ne0 = 128 / elem_size;
const uint32_t ne = na + nb;
if (ne == 0 || ne > max_ne0) {
return false;
}
if (elem_size != 4 && elem_size != 2) {
return false;
}
const uint32_t v_lanes = 128 / elem_size;
const uint32_t g = hex_gcd_u32(ne, v_lanes);
const uint32_t rg = v_lanes / g;
const uint32_t vg = ne / g;
const uint32_t slice = (vtcm_size / n_threads) & ~127u;
const uint32_t total_chunks = total_rows / rg;
const uint32_t chunks_per_dev = (total_chunks + mdev_count - 1) / mdev_count;
const uint32_t max_rows_per_dev = (mdev_count > 1 && total_chunks >= mdev_count)
? (chunks_per_dev * rg + (total_rows % rg))
: total_rows;
const uint32_t rows_per_thread = hex_round_up((max_rows_per_dev + n_threads - 1) / n_threads, rg);
const uint32_t fixed = 3 * vg * 128;
const uint32_t per_row = a_nb1 + (nb > 0 ? (b_dense ? b_nb1 : nb * b_nb1) : 0) + ne * elem_size;
if (slice <= fixed + 2 * per_row * rg) {
return false;
}
const uint32_t avail_per_buf = (slice - fixed) / 2;
uint32_t nr_max = MIN(avail_per_buf / per_row, rows_per_thread);
if (elem_size == 2) {
const uint32_t per_row_region = a_nb1 + (nb > 0 ? (b_dense ? b_nb1 : nb * b_nb1) : 0);
if (per_row_region > 0) {
nr_max = MIN(nr_max, (32768u - 256u) / per_row_region);
}
}
nr_max = (nr_max / rg) * rg;
uint32_t v1 = 0;
uint32_t span1 = 0;
uint32_t region_size = 0;
uint32_t out_off = 0;
uint32_t buf_stride = 0;
while (nr_max > 0) {
v1 = hex_round_up((nr_max - 1) * a_nb1 + na * elem_size, 128);
if (nb > 0) {
if (b_dense) {
span1 = elem_size;
region_size = v1 + (nr_max - 1) * b_nb1 + nb * elem_size;
} else {
span1 = (nr_max - 1) * b_nb1 + elem_size;
region_size = v1 + nb * span1;
}
} else {
span1 = 0;
region_size = v1;
}
out_off = hex_round_up(region_size, 128);
buf_stride = hex_round_up(out_off + nr_max * ne * elem_size, 128);
if ((elem_size != 2 || region_size <= 32768) && fixed + 2 * buf_stride <= slice) {
break;
}
nr_max -= rg;
}
if (nr_max == 0) {
return false;
}
kparams->na = na;
kparams->nb = nb;
kparams->ne = ne;
kparams->rg = rg;
kparams->vg = vg;
kparams->nr_max = nr_max;
kparams->v1 = v1;
kparams->span1 = span1;
kparams->region_size = region_size;
kparams->out_off = out_off;
kparams->buf_stride = buf_stride;
kparams->slice = slice;
kparams->total_rows = total_rows;
kparams->rows_per_thread = rows_per_thread;
kparams->b_dense = b_dense ? 1 : 0;
kparams->pad[0] = 0;
kparams->pad[1] = 0;
kparams->pad[2] = 0;
return true;
}
#ifdef __cplusplus
struct ggml_tensor;
static inline bool htp_gather_rows_tensor_outer_contiguous(const struct ggml_tensor * t) {
size_t next_nb = ((const size_t *) t->nb)[1] * ((const int64_t *) t->ne)[1];
for (int i = 2; i < GGML_MAX_DIMS; i++) {
if (((const int64_t *) t->ne)[i] != 1 && ((const size_t *) t->nb)[i] != next_nb) {
return false;
}
next_nb *= ((const int64_t *) t->ne)[i];
}
return true;
}
#else
struct htp_ops_context;
struct htp_tensor;
int htp_gather_rows(
struct htp_ops_context * octx,
const struct htp_tensor * a,
const struct htp_tensor * b,
const struct htp_gather_rows_params * kparams,
uint32_t n_threads);
#endif
#endif // HTP_GATHER_ROWS_H
+10 -10
View File
@@ -287,15 +287,15 @@ AEEResult htp_iface_munmap(remote_handle64 handle, uint32 fd) {
}
static void vtcm_acquire(struct htp_context * ctx) {
if (!ctx->vtcm_valid) {
if (!atomic_load(&ctx->vtcm_valid)) {
int err = HAP_compute_res_acquire_cached(ctx->vtcm_rctx, 10000000u);
if (err != 0) {
FARF(ERROR, "ggml-hex: failed to acquire VTCM: 0x%08x", (unsigned)err);
abort();
}
ctx->vtcm_needs_release = false;
ctx->vtcm_valid = true;
atomic_store(&ctx->vtcm_needs_release, false);
atomic_store(&ctx->vtcm_valid, true);
// Drop the priority to make sure we get the release callback from other GGML-HTP and QNN-HTP sessions
HAP_compute_res_update_priority(ctx->vtcm_rctx, ctx->thread_prio + 10);
@@ -303,16 +303,16 @@ static void vtcm_acquire(struct htp_context * ctx) {
}
static void vtcm_release(struct htp_context * ctx) {
if (ctx->vtcm_valid) {
ctx->vtcm_valid = false;
ctx->vtcm_needs_release = false;
if (atomic_load(&ctx->vtcm_valid)) {
atomic_store(&ctx->vtcm_valid, false);
atomic_store(&ctx->vtcm_needs_release, false);
HAP_compute_res_release_cached(ctx->vtcm_rctx);
}
}
static int vtcm_release_callback(unsigned int rctx, void * state) {
struct htp_context * ctx = (struct htp_context *) state;
ctx->vtcm_needs_release = true;
atomic_store(&ctx->vtcm_needs_release, true);
return 0;
}
@@ -345,8 +345,8 @@ static int vtcm_alloc(struct htp_context * ctx) {
ctx->vtcm_base = (uint8_t *) vtcm_ptr;
ctx->vtcm_size = vtcm_size;
ctx->vtcm_rctx = rctx;
ctx->vtcm_valid = false;
ctx->vtcm_needs_release = false;
atomic_store(&ctx->vtcm_valid, false);
atomic_store(&ctx->vtcm_needs_release, false);
return 0;
}
@@ -1309,7 +1309,7 @@ static void process_ops(struct htp_context * ctx) {
vtcm_acquire(ctx);
while (!ctx->vtcm_needs_release && !atomic_load(&ctx->killed)) {
while (!atomic_load(&ctx->vtcm_needs_release) && !atomic_load(&ctx->killed)) {
struct htp_opbatch_req req;
uint32_t r_size = sizeof(req);
+88 -38
View File
@@ -10,6 +10,7 @@ import subprocess
import platform
import shutil
import logging
import glob
from sdk import validate_windows_sdks
@@ -51,6 +52,8 @@ def main():
parser.add_argument("--no-docker", action="store_true", help="Build natively on the host instead of in a docker container")
parser.add_argument("--preset", help="Override the CMake preset to use")
parser.add_argument("--debug", action="store_true", help="Build in debug mode (uses -debug presets instead of -release)")
parser.add_argument("--hex-only", "--hexagon-only", action="store_true", help="Only build and push ggml-hexagon and libggml-htp libraries")
parser.add_argument("--hex-arch", choices=["v73", "v75", "v79", "v81", "73", "75", "79", "81"], help="Specific Hexagon DSP architecture to build with --hex-only (default: all)")
# Push options
parser.add_argument("--push", action="store_true", help="Push built package to the target device via ADB or SSH/SCP")
@@ -129,6 +132,16 @@ def main():
jobs = args.jobs if args.jobs else os.cpu_count() or 4
hex_arch = None
if args.hex_arch:
hex_arch = args.hex_arch if args.hex_arch.startswith("v") else f"v{args.hex_arch}"
if args.hex_only:
if hex_arch:
hex_targets = ["ggml-hexagon", f"htp-{hex_arch}"]
else:
hex_targets = ["ggml-hexagon", "htp-v73", "htp-v75", "htp-v79", "htp-v81"]
if args.no_docker:
# Native/local host build
logger.info("Running native/local CMake build...")
@@ -143,7 +156,10 @@ def main():
sys.exit(res.returncode)
# Build
build_cmd = ["cmake", "--build", build_dir, "-j", str(jobs)]
build_cmd = ["cmake", "--build", build_dir]
if args.hex_only:
build_cmd += ["--target"] + hex_targets
build_cmd += ["-j", str(jobs)]
logger.info(f"+ {' '.join(build_cmd)}")
res = subprocess.run(build_cmd, cwd=repo_root)
if res.returncode != 0:
@@ -165,9 +181,10 @@ def main():
install_prefix_container = f"/workspace/{install_dir}/llama.cpp"
target_args = f"--target {' '.join(hex_targets)} " if args.hex_only else ""
build_sh_cmd = (
f"cmake --preset {preset} -B /workspace/{build_dir} && "
f"cmake --build /workspace/{build_dir} -j {jobs} && "
f"cmake --build /workspace/{build_dir} {target_args}-j {jobs} && "
f"cmake --install /workspace/{build_dir} --prefix {install_prefix_container}"
)
@@ -206,58 +223,91 @@ def main():
sub_items = [item for item in os.listdir(src_path) if not item.startswith(".")]
if args.hex_only:
lib_dir = os.path.join(src_path, "lib")
hex_libs = glob.glob(os.path.join(lib_dir, "libggml-hexagon*"))
if hex_arch:
hex_libs += glob.glob(os.path.join(lib_dir, f"libggml-htp-{hex_arch}*"))
else:
hex_libs += glob.glob(os.path.join(lib_dir, "libggml-htp*"))
if not hex_libs:
logger.error(f"Error: no hexagon libraries found in {lib_dir} to push.")
sys.exit(1)
if target_type == "android":
logger.info("\nPushing built artifacts to Android device via ADB...")
adb_cmd = ["adb"]
if target_val: # serial
adb_cmd += ["-s", target_val]
# Clean stale package files on device
if sub_items:
clean_paths = " ".join(f"{target_dir}/{item}" for item in sub_items)
clean_cmd = adb_cmd + ["shell", f"rm -rf {clean_paths}"]
logger.info(f"+ {' '.join(clean_cmd)}")
subprocess.run(clean_cmd)
if args.hex_only:
logger.info("\nPushing Hexagon libraries to Android device via ADB...")
target_lib_dir = f"{target_dir}/lib"
for lib in hex_libs:
push_cmd = adb_cmd + ["push", lib, f"{target_lib_dir}/"]
logger.info(f"+ {' '.join(push_cmd)}")
res = subprocess.run(push_cmd)
if res.returncode != 0:
logger.error("ADB push failed.")
sys.exit(res.returncode)
logger.info("Hexagon libraries deployed successfully!")
else:
# Clean stale package files on device
if sub_items:
clean_paths = " ".join(f"{target_dir}/{item}" for item in sub_items)
clean_cmd = adb_cmd + ["shell", f"rm -rf {clean_paths}"]
logger.info(f"+ {' '.join(clean_cmd)}")
subprocess.run(clean_cmd)
# Android destination directory is target_dir
push_cmd = adb_cmd + ["push", os.path.join(src_path, "."), target_dir]
logger.info(f"+ {' '.join(push_cmd)}")
res = subprocess.run(push_cmd)
if res.returncode != 0:
logger.error("ADB push failed.")
sys.exit(res.returncode)
# Android destination directory is target_dir
push_cmd = adb_cmd + ["push", os.path.join(src_path, "."), target_dir]
logger.info(f"+ {' '.join(push_cmd)}")
res = subprocess.run(push_cmd)
if res.returncode != 0:
logger.error("ADB push failed.")
sys.exit(res.returncode)
chmod_cmd = adb_cmd + ["shell", f"chmod -R 755 {target_dir}/bin 2>/dev/null || true"]
logger.info(f"+ {' '.join(chmod_cmd)}")
subprocess.run(chmod_cmd)
logger.info("ADB push completed successfully!")
chmod_cmd = adb_cmd + ["shell", f"chmod -R 755 {target_dir}/bin 2>/dev/null || true"]
logger.info(f"+ {' '.join(chmod_cmd)}")
subprocess.run(chmod_cmd)
logger.info("ADB push completed successfully!")
elif target_type == "linux":
ssh_host = target_val
if not ssh_host:
logger.error("Error: SSH host not specified in target (e.g. use linux:user@host, lnx:user@host, or ubuntu:user@host). Cannot deploy.")
sys.exit(1)
logger.info(f"\nDeploying built artifacts to Linux device {ssh_host} via SSH/SCP...")
# Clean stale package files on remote host
if sub_items:
clean_paths = " ".join(f"{target_dir}/{item}" for item in sub_items)
clean_cmd = ["ssh", ssh_host, f"rm -rf {clean_paths}"]
logger.info(f"+ {' '.join(clean_cmd)}")
subprocess.run(clean_cmd)
if args.hex_only:
logger.info(f"\nDeploying Hexagon libraries to Linux device {ssh_host} via SCP...")
target_lib_dir = f"{target_dir}/lib"
deploy_cmd = ["scp"] + hex_libs + [f"{ssh_host}:{target_lib_dir}/"]
logger.info(f"+ {' '.join(deploy_cmd)}")
res = subprocess.run(deploy_cmd)
if res.returncode != 0:
logger.error("SCP deploy failed.")
sys.exit(res.returncode)
logger.info("Hexagon libraries deployed successfully!")
else:
# Clean stale package files on remote host
if sub_items:
clean_paths = " ".join(f"{target_dir}/{item}" for item in sub_items)
clean_cmd = ["ssh", ssh_host, f"rm -rf {clean_paths}"]
logger.info(f"+ {' '.join(clean_cmd)}")
subprocess.run(clean_cmd)
# Deploy to target_dir
deploy_cmd = ["scp", "-r", os.path.join(src_path, "."), f"{ssh_host}:{target_dir}"]
logger.info(f"+ {' '.join(deploy_cmd)}")
res = subprocess.run(deploy_cmd)
if res.returncode != 0:
logger.error("SSH/SCP deploy failed.")
sys.exit(res.returncode)
# Deploy to target_dir
deploy_cmd = ["scp", "-r", os.path.join(src_path, "."), f"{ssh_host}:{target_dir}"]
logger.info(f"+ {' '.join(deploy_cmd)}")
res = subprocess.run(deploy_cmd)
if res.returncode != 0:
logger.error("SSH/SCP deploy failed.")
sys.exit(res.returncode)
chmod_cmd = ["ssh", ssh_host, f"chmod -R 755 {target_dir}/bin 2>/dev/null || true"]
logger.info(f"+ {' '.join(chmod_cmd)}")
subprocess.run(chmod_cmd)
logger.info("SSH/SCP deploy completed successfully!")
chmod_cmd = ["ssh", ssh_host, f"chmod -R 755 {target_dir}/bin 2>/dev/null || true"]
logger.info(f"+ {' '.join(chmod_cmd)}")
subprocess.run(chmod_cmd)
logger.info("SSH/SCP deploy completed successfully!")
elif target_type == "windows":
logger.info("\nPush for Windows on Snapdragon (windows) target is currently a stub.")