hexagon: optimize CPY/CONCAT — round 3, short-rows edition (#30341)
* hexagon: move the converting CPY in blocks of rows per DMA descriptor so short rows are not DMA-latency bound * hex-cpy: cleanup cpy cursor handling * hex-cpy-concat: introduce threaded dma/hvx gather-row helper Co-authored-by: Ruslan Karymov <67752306+karusrus@users.noreply.github.com> * hex-cpy: remove leftover rejections for extended buffers * hex-cpy: relax the gate for fast-path * hex-cpy: remove scalar divs and vtcm access and use explicit gather_sync * hex-cpy: add proper support for mdev work splitting to fastpath * hex-cpy: optimize dense gather-row tasks * hex-cpy: unroll gather-rows loops * hex-cpy: add fastpath for f16 * hex-cpy-concat: support for colapsing 3/4D dims * hex-cpy: double-buffer support * hex-cpy: factor out gather-rows into more of an op by itself * hex-cpy-concat: dedup kernel-params * hex-cpy-concat: step-down nr to fit instead of hard fail * hex-gather: replace div with a mask * hex-gather-rows: cleanup dma pipeline and remove the stalls * hex-gather: specialize compute funcs * hex-gather: reduce scalar reg.spills and func calls * hex-gather: do not recompute rows-per-thread on the npu * hex-build: add support for building/installing just hexagon * hex-cpy-concat: remove redundant barrier in concat and move cpy barrier under fence * hex-gather: annotations for dma-out * hex-mdev: make all key vtcm_acquire sync state atomic (fixes random hangs in mdev setups) --------- Co-authored-by: Yiwei Shao <yiwei@aizip.ai> Co-authored-by: Ruslan Karymov <67752306+karusrus@users.noreply.github.com>
This commit is contained in:
10 files changed
+818
-124
No files matched your search
@@ -6593,6 +6593,37 @@ static bool ggml_hexagon_precompute_concat_params(
|
||||
}
|
||||
}
|
||||
|
||||
const uint32_t max_gather_ne0 = 128 / type_size;
|
||||
const uint32_t total_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]);
|
||||
const bool b_dense = (src1->nb[0] == type_size);
|
||||
const bool gather_strides_ok =
|
||||
src0->nb[1] >= (size_t) src0->ne[0] * type_size &&
|
||||
(b_dense ? src1->nb[1] >= (size_t) src1->ne[0] * type_size
|
||||
: src1->nb[0] >= (size_t) src1->ne[1] * type_size);
|
||||
|
||||
if (dim == 0 &&
|
||||
(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16) &&
|
||||
dst->ne[0] <= max_gather_ne0 &&
|
||||
dst->nb[0] == type_size && dst->nb[1] == (size_t) dst->ne[0] * type_size &&
|
||||
src0->nb[0] == type_size &&
|
||||
(src1->nb[0] == type_size || src1->nb[1] == type_size) &&
|
||||
gather_strides_ok &&
|
||||
htp_gather_rows_tensor_outer_contiguous(dst) &&
|
||||
htp_gather_rows_tensor_outer_contiguous(src0) &&
|
||||
htp_gather_rows_tensor_outer_contiguous(src1)) {
|
||||
|
||||
if (htp_gather_rows_solve_layout(&kparams->u.gather,
|
||||
(uint32_t) src0->ne[0], (uint32_t) src0->nb[1],
|
||||
(uint32_t) src1->ne[0], (uint32_t) src1->nb[1], b_dense,
|
||||
type_size, total_rows,
|
||||
sess->n_threads, sess->mdev.count, sess->vtcm_size)) {
|
||||
kparams->kernel_type = HTP_CONCAT_KERNEL_GATHER_ROWS;
|
||||
kparams->n_threads = (uint8_t) sess->n_threads;
|
||||
kparams->vtcm_size = kparams->u.gather.slice * sess->n_threads;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
const bool dma_strides_ok = (src0->nb[0] == type_size && src1->nb[0] == type_size && dst->nb[0] == type_size);
|
||||
|
||||
if (dma_strides_ok) {
|
||||
@@ -6616,11 +6647,11 @@ static bool ggml_hexagon_precompute_concat_params(
|
||||
return false;
|
||||
}
|
||||
|
||||
kparams->kernel_type = HTP_CONCAT_KERNEL_TRANSPOSED;
|
||||
kparams->n_threads = n_threads;
|
||||
kparams->vtcm_size = layout.total_bytes;
|
||||
kparams->spad0_size_per_thread = layout.src0_spad_size_per_thread;
|
||||
kparams->spad1_size_per_thread = layout.src1_spad_size_per_thread;
|
||||
kparams->kernel_type = HTP_CONCAT_KERNEL_TRANSPOSED;
|
||||
kparams->n_threads = n_threads;
|
||||
kparams->vtcm_size = layout.total_bytes;
|
||||
kparams->u.transposed.spad0_size_per_thread = layout.src0_spad_size_per_thread;
|
||||
kparams->u.transposed.spad1_size_per_thread = layout.src1_spad_size_per_thread;
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -6706,6 +6737,27 @@ static bool ggml_hexagon_precompute_cpy_params(
|
||||
return true;
|
||||
}
|
||||
|
||||
const uint32_t max_gather_ne0 = 128 / src_type_size;
|
||||
const uint32_t total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);
|
||||
if ((src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) &&
|
||||
dst_is_contiguous && !src_is_contiguous &&
|
||||
src0->nb[0] == src_type_size && src0->ne[0] <= max_gather_ne0 &&
|
||||
src0->nb[1] >= (size_t) src0->ne[0] * src_type_size &&
|
||||
total_rows > 1 &&
|
||||
htp_gather_rows_tensor_outer_contiguous(src0)) {
|
||||
|
||||
if (htp_gather_rows_solve_layout(&kparams->u.gather,
|
||||
(uint32_t) src0->ne[0], (uint32_t) src0->nb[1],
|
||||
0, 0, false,
|
||||
src_type_size, total_rows,
|
||||
sess->n_threads, sess->mdev.count, sess->vtcm_size)) {
|
||||
kparams->kernel_type = HTP_COPY_KERNEL_GATHER_ROWS;
|
||||
kparams->n_threads = (uint8_t) sess->n_threads;
|
||||
kparams->vtcm_size = kparams->u.gather.slice * sess->n_threads;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if (sameshape) {
|
||||
kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_SAMETYPE;
|
||||
kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);
|
||||
@@ -6737,7 +6789,7 @@ static bool ggml_hexagon_precompute_cpy_params(
|
||||
|
||||
const uint32_t n_threads = sess->n_threads > 0 ? (uint32_t) sess->n_threads : 4;
|
||||
struct htp_copy_convert_vtcm_layout layout;
|
||||
htp_copy_convert_vtcm_layout_build(&layout, (uint32_t) src0->ne[0], src_type_size, dst_type_size, n_threads);
|
||||
htp_copy_convert_vtcm_layout_build(&layout, (uint32_t) src0->ne[0], (uint32_t) src0->ne[1], src_type_size, dst_type_size, n_threads);
|
||||
|
||||
if (sess->vtcm_size > 0 && layout.total_bytes > sess->vtcm_size) {
|
||||
return false;
|
||||
@@ -6751,6 +6803,9 @@ static bool ggml_hexagon_precompute_cpy_params(
|
||||
kparams->u.convert.dst_buf_size = layout.dst_buf_size;
|
||||
kparams->u.convert.spad0_size_per_thread = layout.spad0_size_per_thread;
|
||||
kparams->u.convert.spad1_size_per_thread = layout.spad1_size_per_thread;
|
||||
kparams->u.convert.src0_row_stride = layout.src0_row_stride;
|
||||
kparams->u.convert.dst_row_stride = layout.dst_row_stride;
|
||||
kparams->u.convert.blk_rows = layout.blk_rows;
|
||||
kparams->u.convert.div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]);
|
||||
kparams->u.convert.div_ne02_ne01 = init_fastdiv_values((uint32_t) (src0->ne[2] * src0->ne[1]));
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@ add_library(${HTP_LIB} SHARED
|
||||
set-rows-ops.c
|
||||
get-rows-ops.c
|
||||
cpy-ops.c
|
||||
gather-rows.c
|
||||
repeat-ops.c
|
||||
ssm-conv.c
|
||||
cumsum-ops.c
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
#include "htp-vtcm.h"
|
||||
#include "hvx-utils.h"
|
||||
#include "hvx_hexagon_protos.h"
|
||||
#include "gather-rows.h"
|
||||
|
||||
#include <string.h>
|
||||
|
||||
@@ -302,9 +303,9 @@ static int concat_transposed(struct htp_ops_context * octx, const struct htp_con
|
||||
struct htp_concat_context cctx;
|
||||
cctx.octx = octx;
|
||||
cctx.spad0_base = vtcm_base;
|
||||
cctx.spad1_base = vtcm_base + n_threads * kparams->spad0_size_per_thread;
|
||||
cctx.spad0_size_per_thread = kparams->spad0_size_per_thread;
|
||||
cctx.spad1_size_per_thread = kparams->spad1_size_per_thread;
|
||||
cctx.spad1_base = vtcm_base + n_threads * kparams->u.transposed.spad0_size_per_thread;
|
||||
cctx.spad0_size_per_thread = kparams->u.transposed.spad0_size_per_thread;
|
||||
cctx.spad1_size_per_thread = kparams->u.transposed.spad1_size_per_thread;
|
||||
cctx.row_start = row_start;
|
||||
cctx.nrows = nrows;
|
||||
cctx.nplanes = dst->ne[2] * dst->ne[3];
|
||||
@@ -318,10 +319,13 @@ static int concat_transposed(struct htp_ops_context * octx, const struct htp_con
|
||||
|
||||
int op_concat(struct htp_ops_context * octx) {
|
||||
const struct htp_concat_kernel_params * kparams = (const struct htp_concat_kernel_params *) octx->kernel_params;
|
||||
const struct htp_tensor * dst = octx->dst;
|
||||
const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2;
|
||||
const struct htp_tensor * src0 = octx->src[0];
|
||||
const struct htp_tensor * src1 = octx->src[1];
|
||||
const struct htp_tensor * dst = octx->dst;
|
||||
|
||||
int status = HTP_STATUS_OK;
|
||||
const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2;
|
||||
|
||||
switch (kparams->kernel_type) {
|
||||
case HTP_CONCAT_KERNEL_REGULAR:
|
||||
status = concat_regular(octx, kparams->dim, type_size);
|
||||
@@ -331,6 +335,10 @@ int op_concat(struct htp_ops_context * octx) {
|
||||
status = concat_transposed(octx, kparams, type_size);
|
||||
break;
|
||||
|
||||
case HTP_CONCAT_KERNEL_GATHER_ROWS:
|
||||
status = htp_gather_rows(octx, src0, src1, &kparams->u.gather, kparams->n_threads);
|
||||
break;
|
||||
|
||||
default:
|
||||
status = HTP_STATUS_NO_SUPPORT;
|
||||
break;
|
||||
@@ -338,9 +346,5 @@ int op_concat(struct htp_ops_context * octx) {
|
||||
|
||||
htp_ops_context_set_status(octx, status);
|
||||
|
||||
if (octx->ctx->mdev.count > 1) {
|
||||
htp_mdev_group_barrier(octx);
|
||||
}
|
||||
|
||||
return octx->status;
|
||||
}
|
||||
@@ -2,12 +2,19 @@
|
||||
#define HTP_CONCAT_OPS_H
|
||||
|
||||
#include "hex-common.h"
|
||||
#include "gather-rows.h"
|
||||
#include <stdint.h>
|
||||
|
||||
enum htp_concat_kernel_type {
|
||||
HTP_CONCAT_KERNEL_UNSUPPORTED = 0,
|
||||
HTP_CONCAT_KERNEL_REGULAR = 1,
|
||||
HTP_CONCAT_KERNEL_TRANSPOSED = 2,
|
||||
HTP_CONCAT_KERNEL_GATHER_ROWS = 3,
|
||||
};
|
||||
|
||||
struct htp_concat_transposed_params {
|
||||
uint32_t spad0_size_per_thread;
|
||||
uint32_t spad1_size_per_thread;
|
||||
};
|
||||
|
||||
struct htp_concat_kernel_params {
|
||||
@@ -17,8 +24,11 @@ struct htp_concat_kernel_params {
|
||||
uint8_t pad;
|
||||
|
||||
uint32_t vtcm_size;
|
||||
uint32_t spad0_size_per_thread;
|
||||
uint32_t spad1_size_per_thread;
|
||||
|
||||
union {
|
||||
struct htp_concat_transposed_params transposed;
|
||||
struct htp_gather_rows_params gather;
|
||||
} u;
|
||||
};
|
||||
|
||||
#if defined(__cplusplus)
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
#include "htp-ops.h"
|
||||
#include "htp-tensor.h"
|
||||
#include "hvx-utils.h"
|
||||
#include "gather-rows.h"
|
||||
|
||||
struct htp_copy_context {
|
||||
struct htp_ops_context * octx;
|
||||
@@ -141,6 +142,25 @@ DEFINE_CPY_RESHAPE(f32, float, 4)
|
||||
DEFINE_CPY_RESHAPE(f16, __fp16, 2)
|
||||
DEFINE_CPY_RESHAPE(i32, int32_t, 4)
|
||||
|
||||
struct cpy_row_cursor {
|
||||
uint32_t r;
|
||||
uint32_t i01;
|
||||
uint32_t i02;
|
||||
uint32_t i03;
|
||||
};
|
||||
|
||||
static inline void cpy_cursor_advance(struct cpy_row_cursor * c, uint32_t n, uint32_t ne01, uint32_t ne02) {
|
||||
c->r += n;
|
||||
c->i01 += n;
|
||||
if (c->i01 == ne01) {
|
||||
c->i01 = 0;
|
||||
if (++c->i02 == ne02) {
|
||||
c->i02 = 0;
|
||||
c->i03++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#define DEFINE_CPY_CONVERT_SAMESHAPE(NAME, CONV_FUNC) \
|
||||
static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \
|
||||
struct htp_copy_context * ct = (struct htp_copy_context *) data; \
|
||||
@@ -151,7 +171,6 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vo
|
||||
const uint32_t ir0 = ct->row_start + dr * ith; \
|
||||
const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); \
|
||||
if (ir0 >= ir1) return; \
|
||||
const uint32_t nrows_thread = ir1 - ir0; \
|
||||
\
|
||||
dma_queue * dma_q = octx->ctx->dma[ith]; \
|
||||
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
|
||||
@@ -159,6 +178,9 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vo
|
||||
const struct htp_copy_convert_params * cvt = &ct->kparams->u.convert; \
|
||||
const uint32_t src0_buf_size = cvt->src0_buf_size; \
|
||||
const uint32_t dst_buf_size = cvt->dst_buf_size; \
|
||||
const uint32_t src0_row_stride = cvt->src0_row_stride; \
|
||||
const uint32_t dst_row_stride = cvt->dst_row_stride; \
|
||||
const uint32_t blk_rows = cvt->blk_rows; \
|
||||
uint8_t * vtcm_src0_base = ct->vtcm_src0 + ith * cvt->spad0_size_per_thread; \
|
||||
uint8_t * vtcm_dst_base = ct->vtcm_dst + ith * cvt->spad1_size_per_thread; \
|
||||
const uint32_t src0_row_size = ne00 * ct->kparams->src0_type_size; \
|
||||
@@ -170,62 +192,43 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vo
|
||||
uint32_t i02 = fastdiv(rem, &cvt->div_ne01); \
|
||||
uint32_t i01 = rem - i02 * ne01; \
|
||||
\
|
||||
uint32_t f_i01 = i01, f_i02 = i02, f_i03 = i03; \
|
||||
dma_addr_t f_src0_addr = src0->data + f_i01*nb01 + f_i02*nb02 + f_i03*nb03; \
|
||||
struct cpy_row_cursor f = { ir0, i01, i02, i03 }; \
|
||||
struct cpy_row_cursor c = f; \
|
||||
\
|
||||
uint32_t c_i01 = i01, c_i02 = i02, c_i03 = i03; \
|
||||
dma_addr_t c_dst_addr = dst->data + c_i01*nb1 + c_i02*nb2 + c_i03*nb3; \
|
||||
\
|
||||
for (uint32_t r = 0; r < nrows_thread && r < 2; ++r) { \
|
||||
uint8_t * src_spad = vtcm_src0_base + r * src0_buf_size; \
|
||||
uint8_t * dst_spad = vtcm_dst_base + r * dst_buf_size; \
|
||||
for (uint32_t b = 0; b < 2 && f.r < ir1; ++b) { \
|
||||
uint8_t * src_spad = vtcm_src0_base + b * src0_buf_size; \
|
||||
uint8_t * dst_spad = vtcm_dst_base + b * dst_buf_size; \
|
||||
const uint32_t nr = MIN(blk_rows, MIN(ne01 - f.i01, ir1 - f.r)); \
|
||||
dma_addr_t f_src0_addr = src0->data + f.i01*nb01 + f.i02*nb02 + f.i03*nb03; \
|
||||
dma_queue_push(dma_q, dma_make_data(dst->data, dst_spad), \
|
||||
dst_row_size, dst_buf_size, dst_row_size, 0); \
|
||||
dma_queue_push(dma_q, dma_make_data(src_spad, f_src0_addr), \
|
||||
src0_buf_size, src0_row_size, src0_row_size, 1); \
|
||||
f_src0_addr += nb01; \
|
||||
if (++f_i01 == ne01) { \
|
||||
f_i01 = 0; \
|
||||
if (++f_i02 == ne02) { \
|
||||
f_i02 = 0; \
|
||||
f_i03++; \
|
||||
} \
|
||||
f_src0_addr = src0->data + f_i02*nb02 + f_i03*nb03; \
|
||||
} \
|
||||
src0_row_stride, nb01, src0_row_size, nr); \
|
||||
cpy_cursor_advance(&f, nr, ne01, ne02); \
|
||||
} \
|
||||
\
|
||||
for (uint32_t r = 0; r < nrows_thread; ++r) { \
|
||||
for (uint32_t blk = 0; c.r < ir1; ++blk) { \
|
||||
const uint32_t nr = MIN(blk_rows, MIN(ne01 - c.i01, ir1 - c.r)); \
|
||||
uint8_t * dst_spad = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).src; \
|
||||
uint8_t * src_spad = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).dst; \
|
||||
\
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \
|
||||
CONV_FUNC(dst_spad, src_spad, ne00); \
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \
|
||||
\
|
||||
dma_queue_push(dma_q, dma_make_data(c_dst_addr, dst_spad), \
|
||||
dst_row_size, dst_buf_size, dst_row_size, 1); \
|
||||
c_dst_addr += nb1; \
|
||||
if (++c_i01 == ne01) { \
|
||||
c_i01 = 0; \
|
||||
if (++c_i02 == ne02) { \
|
||||
c_i02 = 0; \
|
||||
c_i03++; \
|
||||
} \
|
||||
c_dst_addr = dst->data + c_i02*nb2 + c_i03*nb3; \
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) blk); \
|
||||
for (uint32_t r = 0; r < nr; ++r) { \
|
||||
CONV_FUNC(dst_spad + r * dst_row_stride, src_spad + r * src0_row_stride, ne00); \
|
||||
} \
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) blk); \
|
||||
\
|
||||
if (r + 2 < nrows_thread) { \
|
||||
dma_addr_t c_dst_addr = dst->data + c.i01*nb1 + c.i02*nb2 + c.i03*nb3; \
|
||||
dma_queue_push(dma_q, dma_make_data(c_dst_addr, dst_spad), \
|
||||
nb1, dst_row_stride, dst_row_size, nr); \
|
||||
cpy_cursor_advance(&c, nr, ne01, ne02); \
|
||||
\
|
||||
if (f.r < ir1) { \
|
||||
const uint32_t fnr = MIN(blk_rows, MIN(ne01 - f.i01, ir1 - f.r)); \
|
||||
dma_addr_t f_src0_addr = src0->data + f.i01*nb01 + f.i02*nb02 + f.i03*nb03; \
|
||||
dma_queue_push(dma_q, dma_make_data(src_spad, f_src0_addr), \
|
||||
src0_buf_size, src0_row_size, src0_row_size, 1); \
|
||||
f_src0_addr += nb01; \
|
||||
if (++f_i01 == ne01) { \
|
||||
f_i01 = 0; \
|
||||
if (++f_i02 == ne02) { \
|
||||
f_i02 = 0; \
|
||||
f_i03++; \
|
||||
} \
|
||||
f_src0_addr = src0->data + f_i02*nb02 + f_i03*nb03; \
|
||||
} \
|
||||
src0_row_stride, nb01, src0_row_size, fnr); \
|
||||
cpy_cursor_advance(&f, fnr, ne01, ne02); \
|
||||
} \
|
||||
} \
|
||||
dma_queue_flush(dma_q); \
|
||||
@@ -337,10 +340,6 @@ static int cpy_sameshape_convert(struct htp_ops_context * octx, const struct htp
|
||||
const struct htp_tensor * src0 = octx->src[0];
|
||||
const struct htp_tensor * dst = octx->dst;
|
||||
|
||||
if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) {
|
||||
return HTP_STATUS_NO_SUPPORT;
|
||||
}
|
||||
|
||||
if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
|
||||
return HTP_STATUS_INVAL_PARAMS;
|
||||
}
|
||||
@@ -398,10 +397,6 @@ static int cpy_reshape(struct htp_ops_context * octx, const struct htp_copy_kern
|
||||
const struct htp_tensor * src0 = octx->src[0];
|
||||
const struct htp_tensor * dst = octx->dst;
|
||||
|
||||
if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) {
|
||||
return HTP_STATUS_NO_SUPPORT;
|
||||
}
|
||||
|
||||
if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
|
||||
return HTP_STATUS_INVAL_PARAMS;
|
||||
}
|
||||
@@ -461,6 +456,9 @@ int op_cpy(struct htp_ops_context * octx) {
|
||||
case HTP_COPY_KERNEL_RESHAPE:
|
||||
status = cpy_reshape(octx, kparams);
|
||||
break;
|
||||
case HTP_COPY_KERNEL_GATHER_ROWS:
|
||||
status = htp_gather_rows(octx, octx->src[0], NULL, &kparams->u.gather, kparams->n_threads);
|
||||
break;
|
||||
default:
|
||||
status = HTP_STATUS_NO_SUPPORT;
|
||||
break;
|
||||
@@ -468,11 +466,11 @@ int op_cpy(struct htp_ops_context * octx) {
|
||||
|
||||
htp_ops_context_set_status(octx, status);
|
||||
|
||||
if (octx->ctx->mdev.count > 1) {
|
||||
htp_mdev_group_barrier(octx);
|
||||
}
|
||||
|
||||
if (octx->op == HTP_OP_CPY_FENCE) {
|
||||
if (octx->ctx->mdev.count > 1) {
|
||||
htp_mdev_group_barrier(octx);
|
||||
}
|
||||
|
||||
if (octx->ctx->mdev.idx == 0) {
|
||||
const struct htp_tensor * sync = octx->src[1];
|
||||
const uint32_t seq = (uint32_t) octx->op_params[0];
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
#include "hex-common.h"
|
||||
#include "hex-fastdiv.h"
|
||||
#include "gather-rows.h"
|
||||
#include <stdint.h>
|
||||
|
||||
enum htp_copy_kernel_type {
|
||||
@@ -12,6 +13,7 @@ enum htp_copy_kernel_type {
|
||||
HTP_COPY_KERNEL_SAMESHAPE_CONVERT = 3,
|
||||
HTP_COPY_KERNEL_RESHAPE = 4,
|
||||
HTP_COPY_KERNEL_SCALAR = 5,
|
||||
HTP_COPY_KERNEL_GATHER_ROWS = 6,
|
||||
};
|
||||
|
||||
struct htp_copy_convert_params {
|
||||
@@ -19,6 +21,9 @@ struct htp_copy_convert_params {
|
||||
uint32_t dst_buf_size;
|
||||
uint32_t spad0_size_per_thread;
|
||||
uint32_t spad1_size_per_thread;
|
||||
uint32_t src0_row_stride; // row pitch in the VTCM buffers
|
||||
uint32_t dst_row_stride;
|
||||
uint32_t blk_rows; // rows moved per DMA descriptor
|
||||
struct fastdiv_values div_ne01;
|
||||
struct fastdiv_values div_ne02_ne01;
|
||||
};
|
||||
@@ -45,10 +50,19 @@ struct htp_copy_kernel_params {
|
||||
union {
|
||||
struct htp_copy_convert_params convert;
|
||||
struct htp_copy_reshape_params reshape;
|
||||
struct htp_gather_rows_params gather;
|
||||
} u;
|
||||
};
|
||||
|
||||
// Size of one VTCM staging buffer: big enough that a DMA descriptor moves many short
|
||||
// rows at once. The copy kernels keep four per thread (two in flight each way), about
|
||||
// 1 MB of VTCM with 8 threads.
|
||||
#define HTP_COPY_VTCM_BLK_BYTES (32 * 1024)
|
||||
|
||||
struct htp_copy_convert_vtcm_layout {
|
||||
uint32_t src0_row_stride;
|
||||
uint32_t dst_row_stride;
|
||||
uint32_t blk_rows;
|
||||
uint32_t src0_buf_size;
|
||||
uint32_t dst_buf_size;
|
||||
uint32_t spad0_size_per_thread;
|
||||
@@ -56,15 +70,30 @@ struct htp_copy_convert_vtcm_layout {
|
||||
uint32_t total_bytes;
|
||||
};
|
||||
|
||||
// ne01 bounds a block: rows of one block share the dim-1 stride.
|
||||
static inline void htp_copy_convert_vtcm_layout_build(
|
||||
struct htp_copy_convert_vtcm_layout * layout,
|
||||
uint32_t ne00,
|
||||
uint32_t ne01,
|
||||
uint32_t src_type_size,
|
||||
uint32_t dst_type_size,
|
||||
uint32_t n_threads) {
|
||||
|
||||
layout->src0_buf_size = hex_round_up(ne00 * src_type_size, 256);
|
||||
layout->dst_buf_size = hex_round_up(ne00 * dst_type_size, 256);
|
||||
layout->src0_row_stride = hex_round_up(ne00 * src_type_size, 256);
|
||||
layout->dst_row_stride = hex_round_up(ne00 * dst_type_size, 256);
|
||||
|
||||
const uint32_t max_row = layout->src0_row_stride > layout->dst_row_stride ? layout->src0_row_stride : layout->dst_row_stride;
|
||||
uint32_t blk_rows = HTP_COPY_VTCM_BLK_BYTES / max_row;
|
||||
if (blk_rows > ne01) {
|
||||
blk_rows = ne01;
|
||||
}
|
||||
if (blk_rows == 0) {
|
||||
blk_rows = 1;
|
||||
}
|
||||
layout->blk_rows = blk_rows;
|
||||
|
||||
layout->src0_buf_size = blk_rows * layout->src0_row_stride;
|
||||
layout->dst_buf_size = blk_rows * layout->dst_row_stride;
|
||||
layout->spad0_size_per_thread = 2 * layout->src0_buf_size;
|
||||
layout->spad1_size_per_thread = 2 * layout->dst_buf_size;
|
||||
layout->total_bytes = n_threads * (layout->spad0_size_per_thread + layout->spad1_size_per_thread);
|
||||
|
||||
@@ -0,0 +1,387 @@
|
||||
#include "gather-rows.h"
|
||||
|
||||
#include <assert.h>
|
||||
|
||||
#include "hex-common.h"
|
||||
#include "hex-profile.h"
|
||||
#include "hvx-utils.h"
|
||||
#include "dma-queue.h"
|
||||
#include "htp-ctx.h"
|
||||
#include "htp-ops.h"
|
||||
#include "htp-tensor.h"
|
||||
#include "work-queue.h"
|
||||
|
||||
typedef void (*gather_rows_compute_fn_t)(
|
||||
uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,
|
||||
const struct htp_gather_rows_params * restrict kparams,
|
||||
const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur);
|
||||
|
||||
struct gather_rows_task {
|
||||
struct htp_ops_context * octx;
|
||||
const struct htp_tensor * a;
|
||||
const struct htp_tensor * b;
|
||||
dma_addr_t dst;
|
||||
uint32_t r_start;
|
||||
uint32_t rows;
|
||||
uint32_t rows_per_thread;
|
||||
const struct htp_gather_rows_params * kparams;
|
||||
gather_rows_compute_fn_t compute;
|
||||
};
|
||||
|
||||
static inline __attribute__((always_inline)) void gather_rows_dma_in(
|
||||
dma_queue * q, const struct gather_rows_task * task,
|
||||
uint8_t * region, uint32_t r, uint32_t nr, uint32_t elem_size) {
|
||||
const struct htp_tensor * a = task->a;
|
||||
const struct htp_tensor * b = task->b;
|
||||
const struct htp_gather_rows_params * kparams = task->kparams;
|
||||
|
||||
const size_t a_size = (nr - 1) * a->nb[1] + kparams->na * elem_size;
|
||||
dma_queue_push(q, dma_make_data(region, a->data + r * a->nb[1]), a_size, a_size, a_size, 1);
|
||||
if (b) {
|
||||
if (kparams->b_dense) {
|
||||
const size_t b_size = (nr - 1) * b->nb[1] + kparams->nb * elem_size;
|
||||
dma_queue_push(q, dma_make_data(region + kparams->v1, b->data + r * b->nb[1]), b_size, b_size, b_size, 1);
|
||||
} else {
|
||||
const size_t b_size = (nr - 1) * b->nb[1] + elem_size;
|
||||
for (uint32_t k = 0; k < kparams->nb; k++) {
|
||||
dma_queue_push(q, dma_make_data(region + kparams->v1 + k * kparams->span1,
|
||||
b->data + k * b->nb[0] + r * b->nb[1]), b_size, b_size, b_size, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static inline void gather_rows_dma_out(dma_queue * q, const struct gather_rows_task * task,
|
||||
uint8_t * out, uint32_t r, uint32_t nr, uint32_t elem_size) {
|
||||
const struct htp_gather_rows_params * kparams = task->kparams;
|
||||
const size_t out_size = nr * kparams->ne * elem_size;
|
||||
dma_queue_push(q, dma_make_data(task->dst + r * kparams->ne * elem_size, out), out_size, out_size, out_size, 1);
|
||||
}
|
||||
|
||||
static __attribute__((noinline)) void gather_rows_init_tables_f32(
|
||||
HVX_Vector * tab, HVX_Vector * inc,
|
||||
const struct htp_gather_rows_params * kparams,
|
||||
const struct htp_tensor * a, const struct htp_tensor * b) {
|
||||
|
||||
const int32_t s_a = kparams->rg * a->nb[1];
|
||||
const int32_t s_b = b ? kparams->rg * b->nb[1] : 0;
|
||||
const uint32_t b_nb1 = b ? b->nb[1] : 0;
|
||||
uint32_t row = 0;
|
||||
uint32_t col = 0;
|
||||
int32_t t_buf[32] __attribute__((aligned(128)));
|
||||
int32_t s_buf[32] __attribute__((aligned(128)));
|
||||
for (uint32_t k = 0; k < kparams->vg; k++) {
|
||||
for (uint32_t lane = 0; lane < 32; lane++) {
|
||||
if (col < kparams->na) {
|
||||
t_buf[lane] = row * a->nb[1] + col * 4;
|
||||
s_buf[lane] = s_a;
|
||||
} else {
|
||||
t_buf[lane] = kparams->v1 + (col - kparams->na) * kparams->span1 + row * b_nb1;
|
||||
s_buf[lane] = s_b;
|
||||
}
|
||||
if (++col == kparams->ne) {
|
||||
col = 0;
|
||||
row++;
|
||||
}
|
||||
}
|
||||
tab[k] = *(const HVX_Vector *) t_buf;
|
||||
inc[k] = *(const HVX_Vector *) s_buf;
|
||||
}
|
||||
}
|
||||
|
||||
static __attribute__((noinline)) void gather_rows_init_tables_f16(
|
||||
HVX_Vector * tab, HVX_Vector * inc,
|
||||
const struct htp_gather_rows_params * kparams,
|
||||
const struct htp_tensor * a, const struct htp_tensor * b) {
|
||||
|
||||
const int16_t s_a = (int16_t) (kparams->rg * a->nb[1]);
|
||||
const int16_t s_b = (int16_t) (b ? kparams->rg * b->nb[1] : 0);
|
||||
const uint32_t b_nb1 = b ? b->nb[1] : 0;
|
||||
uint32_t row = 0;
|
||||
uint32_t col = 0;
|
||||
int16_t t_buf[64] __attribute__((aligned(128)));
|
||||
int16_t s_buf[64] __attribute__((aligned(128)));
|
||||
for (uint32_t k = 0; k < kparams->vg; k++) {
|
||||
for (uint32_t lane = 0; lane < 64; lane++) {
|
||||
if (col < kparams->na) {
|
||||
t_buf[lane] = (int16_t) (row * a->nb[1] + col * 2);
|
||||
s_buf[lane] = s_a;
|
||||
} else {
|
||||
t_buf[lane] = (int16_t) (kparams->v1 + (col - kparams->na) * kparams->span1 + row * b_nb1);
|
||||
s_buf[lane] = s_b;
|
||||
}
|
||||
if (++col == kparams->ne) {
|
||||
col = 0;
|
||||
row++;
|
||||
}
|
||||
}
|
||||
tab[k] = *(const HVX_Vector *) t_buf;
|
||||
inc[k] = *(const HVX_Vector *) s_buf;
|
||||
}
|
||||
}
|
||||
|
||||
static void gather_rows_compute_f32_vg1(
|
||||
uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,
|
||||
const struct htp_gather_rows_params * restrict kparams,
|
||||
const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) {
|
||||
(void) cur;
|
||||
const uint32_t region_size = kparams->region_size;
|
||||
HVX_Vector vcur = tab[0];
|
||||
const HVX_Vector vinc = inc[0];
|
||||
#pragma unroll(4)
|
||||
for (uint32_t v = 0; v < nvec; v++) {
|
||||
Q6_vgather_ARMVw((HVX_Vector *) (out + v * 128), (size_t) region, region_size, vcur);
|
||||
vcur = Q6_Vw_vadd_VwVw(vcur, vinc);
|
||||
}
|
||||
}
|
||||
|
||||
static void gather_rows_compute_f32_vgn(
|
||||
uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,
|
||||
const struct htp_gather_rows_params * restrict kparams,
|
||||
const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) {
|
||||
const uint32_t vg = kparams->vg;
|
||||
const uint32_t region_size = kparams->region_size;
|
||||
for (uint32_t k = 0; k < vg; k++) {
|
||||
cur[k] = tab[k];
|
||||
}
|
||||
uint32_t k = 0;
|
||||
#pragma unroll(4)
|
||||
for (uint32_t v = 0; v < nvec; v++) {
|
||||
Q6_vgather_ARMVw((HVX_Vector *) (out + v * 128), (size_t) region, region_size, cur[k]);
|
||||
cur[k] = Q6_Vw_vadd_VwVw(cur[k], inc[k]);
|
||||
if (++k == vg) {
|
||||
k = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void gather_rows_compute_f16_vg1(
|
||||
uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,
|
||||
const struct htp_gather_rows_params * restrict kparams,
|
||||
const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) {
|
||||
(void) cur;
|
||||
const uint32_t region_size = kparams->region_size;
|
||||
HVX_Vector vcur = tab[0];
|
||||
const HVX_Vector vinc = inc[0];
|
||||
#pragma unroll(4)
|
||||
for (uint32_t v = 0; v < nvec; v++) {
|
||||
Q6_vgather_ARMVh((HVX_Vector *) (out + v * 128), (size_t) region, region_size, vcur);
|
||||
vcur = Q6_Vh_vadd_VhVh(vcur, vinc);
|
||||
}
|
||||
}
|
||||
|
||||
static void gather_rows_compute_f16_vgn(
|
||||
uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,
|
||||
const struct htp_gather_rows_params * restrict kparams,
|
||||
const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) {
|
||||
const uint32_t vg = kparams->vg;
|
||||
const uint32_t region_size = kparams->region_size;
|
||||
for (uint32_t k = 0; k < vg; k++) {
|
||||
cur[k] = tab[k];
|
||||
}
|
||||
uint32_t k = 0;
|
||||
#pragma unroll(4)
|
||||
for (uint32_t v = 0; v < nvec; v++) {
|
||||
Q6_vgather_ARMVh((HVX_Vector *) (out + v * 128), (size_t) region, region_size, cur[k]);
|
||||
cur[k] = Q6_Vh_vadd_VhVh(cur[k], inc[k]);
|
||||
if (++k == vg) {
|
||||
k = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void gather_rows_thread_f32(unsigned int nth, unsigned int ith, void * data) {
|
||||
(void) nth;
|
||||
const struct gather_rows_task * task = (const struct gather_rows_task *) data;
|
||||
const struct htp_gather_rows_params * kparams = task->kparams;
|
||||
const struct htp_tensor * a = task->a;
|
||||
const struct htp_tensor * b = task->b;
|
||||
|
||||
const uint32_t r_offset = MIN(ith * task->rows_per_thread, task->rows);
|
||||
const uint32_t r_beg = task->r_start + r_offset;
|
||||
const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_thread, task->rows);
|
||||
if (r_beg >= r_end) {
|
||||
return;
|
||||
}
|
||||
|
||||
struct htp_thread_trace * tr = &task->octx->ctx->trace[ith];
|
||||
|
||||
uint8_t * base = task->octx->ctx->vtcm_base + ith * kparams->slice;
|
||||
HVX_Vector * tab = (HVX_Vector *) base;
|
||||
HVX_Vector * inc = tab + kparams->vg;
|
||||
HVX_Vector * cur = inc + kparams->vg;
|
||||
uint8_t * region_base = (uint8_t *) (cur + kparams->vg);
|
||||
|
||||
uint8_t * region[2] = { region_base, region_base + kparams->buf_stride };
|
||||
uint8_t * out[2] = { region[0] + kparams->out_off, region[1] + kparams->out_off };
|
||||
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);
|
||||
gather_rows_init_tables_f32(tab, inc, kparams, a, b);
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);
|
||||
|
||||
dma_queue * q = task->octx->ctx->dma[ith];
|
||||
const uint32_t n_in = 1 + (task->b ? (kparams->b_dense ? 1 : kparams->nb) : 0);
|
||||
const uint32_t nr_max = kparams->nr_max;
|
||||
const uint32_t nr0 = MIN(nr_max, r_end - r_beg);
|
||||
|
||||
dma_queue_push(q, dma_make_data(task->dst, out[0]), 0, 0, 0, 0); // dummy out
|
||||
gather_rows_dma_in(q, task, region[0], r_beg, nr0, 4);
|
||||
|
||||
const uint32_t r1 = r_beg + nr_max;
|
||||
if (r1 < r_end) {
|
||||
const uint32_t nr1 = MIN(nr_max, r_end - r1);
|
||||
dma_queue_push(q, dma_make_data(task->dst, out[1]), 0, 0, 0, 0); // dummy out
|
||||
gather_rows_dma_in(q, task, region[1], r1, nr1, 4);
|
||||
}
|
||||
|
||||
uint32_t buf = 0;
|
||||
for (uint32_t r = r_beg; r < r_end; r += nr_max) {
|
||||
const uint32_t nr = MIN(nr_max, r_end - r);
|
||||
const uint32_t next_buf = buf ^ 1;
|
||||
|
||||
dma_queue_pop(q); // complete out
|
||||
for (uint32_t i = 0; i < n_in; i++) {
|
||||
dma_queue_pop(q);
|
||||
}
|
||||
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r);
|
||||
const uint32_t nvec = (nr * kparams->ne + 31) >> 5;
|
||||
task->compute(out[buf], region[buf], nvec, kparams, tab, inc, cur);
|
||||
hvx_gather_sync(out[buf]);
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r);
|
||||
|
||||
gather_rows_dma_out(q, task, out[buf], r, nr, 4);
|
||||
|
||||
const uint32_t r_prefetch = r + 2 * nr_max;
|
||||
if (r_prefetch < r_end) {
|
||||
const uint32_t nr_prefetch = MIN(nr_max, r_end - r_prefetch);
|
||||
gather_rows_dma_in(q, task, region[buf], r_prefetch, nr_prefetch, 4);
|
||||
}
|
||||
|
||||
buf = next_buf;
|
||||
}
|
||||
dma_queue_flush(q);
|
||||
}
|
||||
|
||||
static void gather_rows_thread_f16(unsigned int nth, unsigned int ith, void * data) {
|
||||
(void) nth;
|
||||
const struct gather_rows_task * task = (const struct gather_rows_task *) data;
|
||||
const struct htp_gather_rows_params * kparams = task->kparams;
|
||||
const struct htp_tensor * a = task->a;
|
||||
const struct htp_tensor * b = task->b;
|
||||
|
||||
const uint32_t r_offset = MIN(ith * task->rows_per_thread, task->rows);
|
||||
const uint32_t r_beg = task->r_start + r_offset;
|
||||
const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_thread, task->rows);
|
||||
if (r_beg >= r_end) {
|
||||
return;
|
||||
}
|
||||
|
||||
struct htp_thread_trace * tr = &task->octx->ctx->trace[ith];
|
||||
|
||||
uint8_t * base = task->octx->ctx->vtcm_base + ith * kparams->slice;
|
||||
HVX_Vector * tab = (HVX_Vector *) base;
|
||||
HVX_Vector * inc = tab + kparams->vg;
|
||||
HVX_Vector * cur = inc + kparams->vg;
|
||||
uint8_t * region_base = (uint8_t *) (cur + kparams->vg);
|
||||
|
||||
uint8_t * region[2] = { region_base, region_base + kparams->buf_stride };
|
||||
uint8_t * out[2] = { region[0] + kparams->out_off, region[1] + kparams->out_off };
|
||||
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);
|
||||
gather_rows_init_tables_f16(tab, inc, kparams, a, b);
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);
|
||||
|
||||
dma_queue * q = task->octx->ctx->dma[ith];
|
||||
const uint32_t n_in = 1 + (task->b ? (kparams->b_dense ? 1 : kparams->nb) : 0);
|
||||
|
||||
const uint32_t nr_max = kparams->nr_max;
|
||||
const uint32_t nr0 = MIN(nr_max, r_end - r_beg);
|
||||
dma_queue_push(q, dma_make_data(task->dst, out[0]), 0, 0, 0, 0); // dummy out
|
||||
gather_rows_dma_in(q, task, region[0], r_beg, nr0, 2);
|
||||
|
||||
const uint32_t r1 = r_beg + nr_max;
|
||||
if (r1 < r_end) {
|
||||
const uint32_t nr1 = MIN(nr_max, r_end - r1);
|
||||
dma_queue_push(q, dma_make_data(task->dst, out[1]), 0, 0, 0, 0); // dummy out
|
||||
gather_rows_dma_in(q, task, region[1], r1, nr1, 2);
|
||||
}
|
||||
|
||||
uint32_t buf = 0;
|
||||
for (uint32_t r = r_beg; r < r_end; r += nr_max) {
|
||||
const uint32_t nr = MIN(nr_max, r_end - r);
|
||||
const uint32_t next_buf = buf ^ 1;
|
||||
|
||||
dma_queue_pop(q); // complete out
|
||||
for (uint32_t i = 0; i < n_in; i++) {
|
||||
dma_queue_pop(q);
|
||||
}
|
||||
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r);
|
||||
const uint32_t nvec = (nr * kparams->ne + 63) >> 6;
|
||||
task->compute(out[buf], region[buf], nvec, kparams, tab, inc, cur);
|
||||
hvx_gather_sync(out[buf]);
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r);
|
||||
|
||||
gather_rows_dma_out(q, task, out[buf], r, nr, 2);
|
||||
|
||||
const uint32_t r_prefetch = r + 2 * nr_max;
|
||||
if (r_prefetch < r_end) {
|
||||
const uint32_t nr_prefetch = MIN(nr_max, r_end - r_prefetch);
|
||||
gather_rows_dma_in(q, task, region[buf], r_prefetch, nr_prefetch, 2);
|
||||
}
|
||||
|
||||
buf = next_buf;
|
||||
}
|
||||
dma_queue_flush(q);
|
||||
}
|
||||
|
||||
int htp_gather_rows(
|
||||
struct htp_ops_context * octx,
|
||||
const struct htp_tensor * a,
|
||||
const struct htp_tensor * b,
|
||||
const struct htp_gather_rows_params * kparams,
|
||||
uint32_t n_threads) {
|
||||
|
||||
if (!htp_ops_context_set_n_threads(octx, n_threads)) {
|
||||
return HTP_STATUS_INVAL_PARAMS;
|
||||
}
|
||||
|
||||
assert(kparams->slice * octx->n_threads <= octx->ctx->vtcm_size);
|
||||
|
||||
const uint32_t total_rows = kparams->total_rows;
|
||||
uint32_t r_start = 0;
|
||||
uint32_t r_count = total_rows;
|
||||
if (octx->ctx->mdev.count > 1) {
|
||||
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(
|
||||
total_rows, kparams->rg, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
|
||||
r_start = range.start;
|
||||
r_count = range.count;
|
||||
}
|
||||
|
||||
if (r_count == 0) {
|
||||
return HTP_STATUS_OK;
|
||||
}
|
||||
|
||||
const uint32_t elem_size = (a->type == HTP_TYPE_F16) ? 2 : 4;
|
||||
|
||||
struct gather_rows_task task;
|
||||
task.octx = octx;
|
||||
task.a = a;
|
||||
task.b = b;
|
||||
task.dst = octx->dst->data;
|
||||
task.r_start = r_start;
|
||||
task.rows = r_count;
|
||||
task.kparams = kparams;
|
||||
task.rows_per_thread = kparams->rows_per_thread;
|
||||
|
||||
if (elem_size == 2) {
|
||||
task.compute = (kparams->vg == 1) ? gather_rows_compute_f16_vg1 : gather_rows_compute_f16_vgn;
|
||||
} else {
|
||||
task.compute = (kparams->vg == 1) ? gather_rows_compute_f32_vg1 : gather_rows_compute_f32_vgn;
|
||||
}
|
||||
|
||||
work_queue_func_t worker_func = (elem_size == 2) ? gather_rows_thread_f16 : gather_rows_thread_f32;
|
||||
work_queue_run(octx->ctx->work_queue, worker_func, &task, octx->n_threads);
|
||||
return HTP_STATUS_OK;
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
#ifndef HTP_GATHER_ROWS_H
|
||||
#define HTP_GATHER_ROWS_H
|
||||
|
||||
#include "hex-common.h"
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
#ifndef GGML_MAX_DIMS
|
||||
#define GGML_MAX_DIMS 4
|
||||
#endif
|
||||
|
||||
struct htp_gather_rows_params {
|
||||
uint32_t na;
|
||||
uint32_t nb;
|
||||
uint32_t ne;
|
||||
uint32_t rg;
|
||||
uint32_t vg;
|
||||
uint32_t nr_max;
|
||||
uint32_t v1;
|
||||
uint32_t span1;
|
||||
uint32_t region_size;
|
||||
uint32_t out_off;
|
||||
uint32_t buf_stride;
|
||||
uint32_t slice;
|
||||
uint32_t total_rows;
|
||||
uint32_t rows_per_thread;
|
||||
uint8_t b_dense;
|
||||
uint8_t pad[3];
|
||||
};
|
||||
|
||||
static inline bool htp_gather_rows_solve_layout(
|
||||
struct htp_gather_rows_params * kparams,
|
||||
uint32_t na, uint32_t a_nb1,
|
||||
uint32_t nb, uint32_t b_nb1, bool b_dense,
|
||||
uint32_t elem_size,
|
||||
uint32_t total_rows,
|
||||
uint32_t n_threads,
|
||||
uint32_t mdev_count,
|
||||
uint32_t vtcm_size) {
|
||||
|
||||
const uint32_t max_ne0 = 128 / elem_size;
|
||||
const uint32_t ne = na + nb;
|
||||
if (ne == 0 || ne > max_ne0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (elem_size != 4 && elem_size != 2) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const uint32_t v_lanes = 128 / elem_size;
|
||||
const uint32_t g = hex_gcd_u32(ne, v_lanes);
|
||||
const uint32_t rg = v_lanes / g;
|
||||
const uint32_t vg = ne / g;
|
||||
|
||||
const uint32_t slice = (vtcm_size / n_threads) & ~127u;
|
||||
const uint32_t total_chunks = total_rows / rg;
|
||||
const uint32_t chunks_per_dev = (total_chunks + mdev_count - 1) / mdev_count;
|
||||
const uint32_t max_rows_per_dev = (mdev_count > 1 && total_chunks >= mdev_count)
|
||||
? (chunks_per_dev * rg + (total_rows % rg))
|
||||
: total_rows;
|
||||
const uint32_t rows_per_thread = hex_round_up((max_rows_per_dev + n_threads - 1) / n_threads, rg);
|
||||
|
||||
const uint32_t fixed = 3 * vg * 128;
|
||||
const uint32_t per_row = a_nb1 + (nb > 0 ? (b_dense ? b_nb1 : nb * b_nb1) : 0) + ne * elem_size;
|
||||
if (slice <= fixed + 2 * per_row * rg) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const uint32_t avail_per_buf = (slice - fixed) / 2;
|
||||
uint32_t nr_max = MIN(avail_per_buf / per_row, rows_per_thread);
|
||||
if (elem_size == 2) {
|
||||
const uint32_t per_row_region = a_nb1 + (nb > 0 ? (b_dense ? b_nb1 : nb * b_nb1) : 0);
|
||||
if (per_row_region > 0) {
|
||||
nr_max = MIN(nr_max, (32768u - 256u) / per_row_region);
|
||||
}
|
||||
}
|
||||
nr_max = (nr_max / rg) * rg;
|
||||
|
||||
uint32_t v1 = 0;
|
||||
uint32_t span1 = 0;
|
||||
uint32_t region_size = 0;
|
||||
uint32_t out_off = 0;
|
||||
uint32_t buf_stride = 0;
|
||||
|
||||
while (nr_max > 0) {
|
||||
v1 = hex_round_up((nr_max - 1) * a_nb1 + na * elem_size, 128);
|
||||
if (nb > 0) {
|
||||
if (b_dense) {
|
||||
span1 = elem_size;
|
||||
region_size = v1 + (nr_max - 1) * b_nb1 + nb * elem_size;
|
||||
} else {
|
||||
span1 = (nr_max - 1) * b_nb1 + elem_size;
|
||||
region_size = v1 + nb * span1;
|
||||
}
|
||||
} else {
|
||||
span1 = 0;
|
||||
region_size = v1;
|
||||
}
|
||||
|
||||
out_off = hex_round_up(region_size, 128);
|
||||
buf_stride = hex_round_up(out_off + nr_max * ne * elem_size, 128);
|
||||
|
||||
if ((elem_size != 2 || region_size <= 32768) && fixed + 2 * buf_stride <= slice) {
|
||||
break;
|
||||
}
|
||||
|
||||
nr_max -= rg;
|
||||
}
|
||||
|
||||
if (nr_max == 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
kparams->na = na;
|
||||
kparams->nb = nb;
|
||||
kparams->ne = ne;
|
||||
kparams->rg = rg;
|
||||
kparams->vg = vg;
|
||||
kparams->nr_max = nr_max;
|
||||
kparams->v1 = v1;
|
||||
kparams->span1 = span1;
|
||||
kparams->region_size = region_size;
|
||||
kparams->out_off = out_off;
|
||||
kparams->buf_stride = buf_stride;
|
||||
kparams->slice = slice;
|
||||
kparams->total_rows = total_rows;
|
||||
kparams->rows_per_thread = rows_per_thread;
|
||||
kparams->b_dense = b_dense ? 1 : 0;
|
||||
kparams->pad[0] = 0;
|
||||
kparams->pad[1] = 0;
|
||||
kparams->pad[2] = 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
struct ggml_tensor;
|
||||
static inline bool htp_gather_rows_tensor_outer_contiguous(const struct ggml_tensor * t) {
|
||||
size_t next_nb = ((const size_t *) t->nb)[1] * ((const int64_t *) t->ne)[1];
|
||||
for (int i = 2; i < GGML_MAX_DIMS; i++) {
|
||||
if (((const int64_t *) t->ne)[i] != 1 && ((const size_t *) t->nb)[i] != next_nb) {
|
||||
return false;
|
||||
}
|
||||
next_nb *= ((const int64_t *) t->ne)[i];
|
||||
}
|
||||
return true;
|
||||
}
|
||||
#else
|
||||
struct htp_ops_context;
|
||||
struct htp_tensor;
|
||||
|
||||
int htp_gather_rows(
|
||||
struct htp_ops_context * octx,
|
||||
const struct htp_tensor * a,
|
||||
const struct htp_tensor * b,
|
||||
const struct htp_gather_rows_params * kparams,
|
||||
uint32_t n_threads);
|
||||
#endif
|
||||
|
||||
#endif // HTP_GATHER_ROWS_H
|
||||
@@ -287,15 +287,15 @@ AEEResult htp_iface_munmap(remote_handle64 handle, uint32 fd) {
|
||||
}
|
||||
|
||||
static void vtcm_acquire(struct htp_context * ctx) {
|
||||
if (!ctx->vtcm_valid) {
|
||||
if (!atomic_load(&ctx->vtcm_valid)) {
|
||||
int err = HAP_compute_res_acquire_cached(ctx->vtcm_rctx, 10000000u);
|
||||
if (err != 0) {
|
||||
FARF(ERROR, "ggml-hex: failed to acquire VTCM: 0x%08x", (unsigned)err);
|
||||
abort();
|
||||
}
|
||||
|
||||
ctx->vtcm_needs_release = false;
|
||||
ctx->vtcm_valid = true;
|
||||
atomic_store(&ctx->vtcm_needs_release, false);
|
||||
atomic_store(&ctx->vtcm_valid, true);
|
||||
|
||||
// Drop the priority to make sure we get the release callback from other GGML-HTP and QNN-HTP sessions
|
||||
HAP_compute_res_update_priority(ctx->vtcm_rctx, ctx->thread_prio + 10);
|
||||
@@ -303,16 +303,16 @@ static void vtcm_acquire(struct htp_context * ctx) {
|
||||
}
|
||||
|
||||
static void vtcm_release(struct htp_context * ctx) {
|
||||
if (ctx->vtcm_valid) {
|
||||
ctx->vtcm_valid = false;
|
||||
ctx->vtcm_needs_release = false;
|
||||
if (atomic_load(&ctx->vtcm_valid)) {
|
||||
atomic_store(&ctx->vtcm_valid, false);
|
||||
atomic_store(&ctx->vtcm_needs_release, false);
|
||||
HAP_compute_res_release_cached(ctx->vtcm_rctx);
|
||||
}
|
||||
}
|
||||
|
||||
static int vtcm_release_callback(unsigned int rctx, void * state) {
|
||||
struct htp_context * ctx = (struct htp_context *) state;
|
||||
ctx->vtcm_needs_release = true;
|
||||
atomic_store(&ctx->vtcm_needs_release, true);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -345,8 +345,8 @@ static int vtcm_alloc(struct htp_context * ctx) {
|
||||
ctx->vtcm_base = (uint8_t *) vtcm_ptr;
|
||||
ctx->vtcm_size = vtcm_size;
|
||||
ctx->vtcm_rctx = rctx;
|
||||
ctx->vtcm_valid = false;
|
||||
ctx->vtcm_needs_release = false;
|
||||
atomic_store(&ctx->vtcm_valid, false);
|
||||
atomic_store(&ctx->vtcm_needs_release, false);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1309,7 +1309,7 @@ static void process_ops(struct htp_context * ctx) {
|
||||
|
||||
vtcm_acquire(ctx);
|
||||
|
||||
while (!ctx->vtcm_needs_release && !atomic_load(&ctx->killed)) {
|
||||
while (!atomic_load(&ctx->vtcm_needs_release) && !atomic_load(&ctx->killed)) {
|
||||
struct htp_opbatch_req req;
|
||||
uint32_t r_size = sizeof(req);
|
||||
|
||||
|
||||
+88
-38
@@ -10,6 +10,7 @@ import subprocess
|
||||
import platform
|
||||
import shutil
|
||||
import logging
|
||||
import glob
|
||||
|
||||
from sdk import validate_windows_sdks
|
||||
|
||||
@@ -51,6 +52,8 @@ def main():
|
||||
parser.add_argument("--no-docker", action="store_true", help="Build natively on the host instead of in a docker container")
|
||||
parser.add_argument("--preset", help="Override the CMake preset to use")
|
||||
parser.add_argument("--debug", action="store_true", help="Build in debug mode (uses -debug presets instead of -release)")
|
||||
parser.add_argument("--hex-only", "--hexagon-only", action="store_true", help="Only build and push ggml-hexagon and libggml-htp libraries")
|
||||
parser.add_argument("--hex-arch", choices=["v73", "v75", "v79", "v81", "73", "75", "79", "81"], help="Specific Hexagon DSP architecture to build with --hex-only (default: all)")
|
||||
|
||||
# Push options
|
||||
parser.add_argument("--push", action="store_true", help="Push built package to the target device via ADB or SSH/SCP")
|
||||
@@ -129,6 +132,16 @@ def main():
|
||||
|
||||
jobs = args.jobs if args.jobs else os.cpu_count() or 4
|
||||
|
||||
hex_arch = None
|
||||
if args.hex_arch:
|
||||
hex_arch = args.hex_arch if args.hex_arch.startswith("v") else f"v{args.hex_arch}"
|
||||
|
||||
if args.hex_only:
|
||||
if hex_arch:
|
||||
hex_targets = ["ggml-hexagon", f"htp-{hex_arch}"]
|
||||
else:
|
||||
hex_targets = ["ggml-hexagon", "htp-v73", "htp-v75", "htp-v79", "htp-v81"]
|
||||
|
||||
if args.no_docker:
|
||||
# Native/local host build
|
||||
logger.info("Running native/local CMake build...")
|
||||
@@ -143,7 +156,10 @@ def main():
|
||||
sys.exit(res.returncode)
|
||||
|
||||
# Build
|
||||
build_cmd = ["cmake", "--build", build_dir, "-j", str(jobs)]
|
||||
build_cmd = ["cmake", "--build", build_dir]
|
||||
if args.hex_only:
|
||||
build_cmd += ["--target"] + hex_targets
|
||||
build_cmd += ["-j", str(jobs)]
|
||||
logger.info(f"+ {' '.join(build_cmd)}")
|
||||
res = subprocess.run(build_cmd, cwd=repo_root)
|
||||
if res.returncode != 0:
|
||||
@@ -165,9 +181,10 @@ def main():
|
||||
|
||||
install_prefix_container = f"/workspace/{install_dir}/llama.cpp"
|
||||
|
||||
target_args = f"--target {' '.join(hex_targets)} " if args.hex_only else ""
|
||||
build_sh_cmd = (
|
||||
f"cmake --preset {preset} -B /workspace/{build_dir} && "
|
||||
f"cmake --build /workspace/{build_dir} -j {jobs} && "
|
||||
f"cmake --build /workspace/{build_dir} {target_args}-j {jobs} && "
|
||||
f"cmake --install /workspace/{build_dir} --prefix {install_prefix_container}"
|
||||
)
|
||||
|
||||
@@ -206,58 +223,91 @@ def main():
|
||||
|
||||
sub_items = [item for item in os.listdir(src_path) if not item.startswith(".")]
|
||||
|
||||
if args.hex_only:
|
||||
lib_dir = os.path.join(src_path, "lib")
|
||||
hex_libs = glob.glob(os.path.join(lib_dir, "libggml-hexagon*"))
|
||||
if hex_arch:
|
||||
hex_libs += glob.glob(os.path.join(lib_dir, f"libggml-htp-{hex_arch}*"))
|
||||
else:
|
||||
hex_libs += glob.glob(os.path.join(lib_dir, "libggml-htp*"))
|
||||
|
||||
if not hex_libs:
|
||||
logger.error(f"Error: no hexagon libraries found in {lib_dir} to push.")
|
||||
sys.exit(1)
|
||||
|
||||
if target_type == "android":
|
||||
logger.info("\nPushing built artifacts to Android device via ADB...")
|
||||
adb_cmd = ["adb"]
|
||||
if target_val: # serial
|
||||
adb_cmd += ["-s", target_val]
|
||||
|
||||
# Clean stale package files on device
|
||||
if sub_items:
|
||||
clean_paths = " ".join(f"{target_dir}/{item}" for item in sub_items)
|
||||
clean_cmd = adb_cmd + ["shell", f"rm -rf {clean_paths}"]
|
||||
logger.info(f"+ {' '.join(clean_cmd)}")
|
||||
subprocess.run(clean_cmd)
|
||||
if args.hex_only:
|
||||
logger.info("\nPushing Hexagon libraries to Android device via ADB...")
|
||||
target_lib_dir = f"{target_dir}/lib"
|
||||
for lib in hex_libs:
|
||||
push_cmd = adb_cmd + ["push", lib, f"{target_lib_dir}/"]
|
||||
logger.info(f"+ {' '.join(push_cmd)}")
|
||||
res = subprocess.run(push_cmd)
|
||||
if res.returncode != 0:
|
||||
logger.error("ADB push failed.")
|
||||
sys.exit(res.returncode)
|
||||
logger.info("Hexagon libraries deployed successfully!")
|
||||
else:
|
||||
# Clean stale package files on device
|
||||
if sub_items:
|
||||
clean_paths = " ".join(f"{target_dir}/{item}" for item in sub_items)
|
||||
clean_cmd = adb_cmd + ["shell", f"rm -rf {clean_paths}"]
|
||||
logger.info(f"+ {' '.join(clean_cmd)}")
|
||||
subprocess.run(clean_cmd)
|
||||
|
||||
# Android destination directory is target_dir
|
||||
push_cmd = adb_cmd + ["push", os.path.join(src_path, "."), target_dir]
|
||||
logger.info(f"+ {' '.join(push_cmd)}")
|
||||
res = subprocess.run(push_cmd)
|
||||
if res.returncode != 0:
|
||||
logger.error("ADB push failed.")
|
||||
sys.exit(res.returncode)
|
||||
# Android destination directory is target_dir
|
||||
push_cmd = adb_cmd + ["push", os.path.join(src_path, "."), target_dir]
|
||||
logger.info(f"+ {' '.join(push_cmd)}")
|
||||
res = subprocess.run(push_cmd)
|
||||
if res.returncode != 0:
|
||||
logger.error("ADB push failed.")
|
||||
sys.exit(res.returncode)
|
||||
|
||||
chmod_cmd = adb_cmd + ["shell", f"chmod -R 755 {target_dir}/bin 2>/dev/null || true"]
|
||||
logger.info(f"+ {' '.join(chmod_cmd)}")
|
||||
subprocess.run(chmod_cmd)
|
||||
logger.info("ADB push completed successfully!")
|
||||
chmod_cmd = adb_cmd + ["shell", f"chmod -R 755 {target_dir}/bin 2>/dev/null || true"]
|
||||
logger.info(f"+ {' '.join(chmod_cmd)}")
|
||||
subprocess.run(chmod_cmd)
|
||||
logger.info("ADB push completed successfully!")
|
||||
|
||||
elif target_type == "linux":
|
||||
ssh_host = target_val
|
||||
if not ssh_host:
|
||||
logger.error("Error: SSH host not specified in target (e.g. use linux:user@host, lnx:user@host, or ubuntu:user@host). Cannot deploy.")
|
||||
sys.exit(1)
|
||||
logger.info(f"\nDeploying built artifacts to Linux device {ssh_host} via SSH/SCP...")
|
||||
|
||||
# Clean stale package files on remote host
|
||||
if sub_items:
|
||||
clean_paths = " ".join(f"{target_dir}/{item}" for item in sub_items)
|
||||
clean_cmd = ["ssh", ssh_host, f"rm -rf {clean_paths}"]
|
||||
logger.info(f"+ {' '.join(clean_cmd)}")
|
||||
subprocess.run(clean_cmd)
|
||||
if args.hex_only:
|
||||
logger.info(f"\nDeploying Hexagon libraries to Linux device {ssh_host} via SCP...")
|
||||
target_lib_dir = f"{target_dir}/lib"
|
||||
deploy_cmd = ["scp"] + hex_libs + [f"{ssh_host}:{target_lib_dir}/"]
|
||||
logger.info(f"+ {' '.join(deploy_cmd)}")
|
||||
res = subprocess.run(deploy_cmd)
|
||||
if res.returncode != 0:
|
||||
logger.error("SCP deploy failed.")
|
||||
sys.exit(res.returncode)
|
||||
logger.info("Hexagon libraries deployed successfully!")
|
||||
else:
|
||||
# Clean stale package files on remote host
|
||||
if sub_items:
|
||||
clean_paths = " ".join(f"{target_dir}/{item}" for item in sub_items)
|
||||
clean_cmd = ["ssh", ssh_host, f"rm -rf {clean_paths}"]
|
||||
logger.info(f"+ {' '.join(clean_cmd)}")
|
||||
subprocess.run(clean_cmd)
|
||||
|
||||
# Deploy to target_dir
|
||||
deploy_cmd = ["scp", "-r", os.path.join(src_path, "."), f"{ssh_host}:{target_dir}"]
|
||||
logger.info(f"+ {' '.join(deploy_cmd)}")
|
||||
res = subprocess.run(deploy_cmd)
|
||||
if res.returncode != 0:
|
||||
logger.error("SSH/SCP deploy failed.")
|
||||
sys.exit(res.returncode)
|
||||
# Deploy to target_dir
|
||||
deploy_cmd = ["scp", "-r", os.path.join(src_path, "."), f"{ssh_host}:{target_dir}"]
|
||||
logger.info(f"+ {' '.join(deploy_cmd)}")
|
||||
res = subprocess.run(deploy_cmd)
|
||||
if res.returncode != 0:
|
||||
logger.error("SSH/SCP deploy failed.")
|
||||
sys.exit(res.returncode)
|
||||
|
||||
chmod_cmd = ["ssh", ssh_host, f"chmod -R 755 {target_dir}/bin 2>/dev/null || true"]
|
||||
logger.info(f"+ {' '.join(chmod_cmd)}")
|
||||
subprocess.run(chmod_cmd)
|
||||
logger.info("SSH/SCP deploy completed successfully!")
|
||||
chmod_cmd = ["ssh", ssh_host, f"chmod -R 755 {target_dir}/bin 2>/dev/null || true"]
|
||||
logger.info(f"+ {' '.join(chmod_cmd)}")
|
||||
subprocess.run(chmod_cmd)
|
||||
logger.info("SSH/SCP deploy completed successfully!")
|
||||
|
||||
elif target_type == "windows":
|
||||
logger.info("\nPush for Windows on Snapdragon (windows) target is currently a stub.")
|
||||
|
||||
Reference in new issue
Block a user