hexagon: support for multi-NPU devices (IQ9, IQ10) and fully asynchronous backend (#26501)
* hexagon: use non-host bufs by default and make the backend fully async * hex-hb: remove optional hostbuf support and fix async copy * hex-unary: relax supported unary check * hex-bufs: use same get_alignment for host bufs * snapdragon: bump android_platform to 34 * hex-rows: super hacky get/set rows for q8_0 * hex-get-rows: fix q8_0 * hex-get-rows: supprot for f16 and cleanup for q8_0 * hex-get-rows: generic macros and specialized thread funcs * hex-get-rows: add DMA pipeline, vtcm_layout and kernel params * hex-set-rows: fix q8_0 support, add dma and tracing * hex-tests: override nmse threshold for HTP of Q8_0 quants * hex-fa: add support for Q8_0 with inplace dequantizers * hex-get-rows: simplify type dispatch * hex-rows: simplify GET/SET_ROWS DMA pipeline * hex-async: add events, set/get-tensor-async and rest of the async api support * hex-repack: use slice instead of expert in repack functions * hex-cpy: update event/async-cpy logging * hex-set-rows: optimize smaller tensors * hex-geglu: fix perf regression with larger tensors * hex-get-rows: add missing header * hex-set-rows: add missing header * hex-bufs: ressurect GGML_HEXAGON_HOSTBUF but disable it by default * hexagon: do not reject ops with non-heaxon buffers * hex-get-rows: apply >=32 restriction only for q8_0 * hex-res: bump vtcm acquire timeout to 10 seconds * hex-bufs: add support for cloning buffers between sessions to speed up tensor copies * hex-async: rework event recording and batch flushing and integrate with meta backend * hex-bufs: improved handling of repacked tensors * hex-repack: handle get_tensor_2d offsets * hex-dev: add support for devices with multiple NPUs * hex-sync: add support for sync tokens to synchronize npu devices for async splits * hex-mmap: cleanup mmap calls and add a retry for robustness * hex-sync: add failsafe if sync wait gets stuck * hex-sync: use sync_seq to check for completed events * hex-sync: rotate tokens for extra robustness * hex-devs: add supprot for legacy device names for now * hex-bufs: add support for auto-cloning buffers from diff sessions * hex-fusion: simplify and optimize htp-opnode fusion handling * hex-sync: override opnode name so that it shows up in the profiles * hex-trace: update scripts to handle multiple devices * hex-sync: bump the size of the opbatch queue and number of sync tokens * hex-cpy-sync: do not explicitly flush opbatches in cpy_tensor_async and add support for cpy-dma * hex-sync: add graph-flush threshold to avoid single op batches * hex-sync: add sync_peer so that we can flush peers we depend on during cross-device ops * hex-bufs: introduce tensor->extra and shadow_bufs for repacking * hex-l2: flush tiny tensors inline * hex-sync: use explicit l2flush for sync tokens * hex-extra: track weight flags via tensor extra * hex-fence: rename sync to fence * hex-repack: proper handling of set-tensor-2d in the shadow_buf * hex-trace: remove obsolete opstage mask that we used for profiling * hex-env: remove obsolete use_hmx variable * hexagon: new unified run.py and build.py and updated docs * snapdragon: update run script to auto-escapt test-backend-op -p argument * hex-scripts: fix trailing spaces * hex-scripts: fix flake8 warnings * snapdragon: cleanup dst lib/bin dirs before copying new build * hex-ops: add support for allreduce * hex-ar: improved allreduce with dma pipeline * hex-ar: align macros * hex-ar: consistent use of fence_seq * hex-ar: add AR_SELECT env var to select ALLREDUCE kernel or fallback * hex-ar: add proper synchronize handling for ALLREDUCE * hex-opbatch: looks like we now just rely on backend.synchronise to flush the batches, no need to flush them by threshold * hex-ar: bump block size to improve dma efficiency * hex-ar: fused ALLREDUCE+ADD * hex-ar: cleaner fence buffer management * hex-ar: futher allreduce tweaking to remove race conditions * hex-ar: add simple solver and remove non-dma kernels * hex-ar: add row-broadcast to fuse with bias ADD * hex-fence: pass seq numbers via op_params * hex-ar: allow for both entry/exit seq for completing entry wait * hex-ar: align macros * hex-ar: do not refetch broadcast row * hex-fusion: move all fusion into opbatch::add_op for consistency with ALLREDUCE and things * hex-fusion: fix incorrect MUL_MAT reordering * hex-mm: make fused 2x and 3x matmuls more generic * hex-fusion: move tensor fusion tagging to graph_compute * hexagon: make sure to copy tensor->extra by value * hex-get-rows: fix offset calc with row-chunking * hex-repack: get_tensor_2d fixes for non-zero offsets * snapdragon: make profile/trace scripts more robust and donot mix stdout/stderr by default * hex-devices: use legacy device nameing by default to ease the transition * hex-devices: hardcode CDSP domain IDs for current devices for now * hex-optrace: improve multi-NPU timestamp alignment and overall handling of cycle values * hex-optrace: more robust handling of the fence events
This commit is contained in:
Max Krasnyansky
@@ -12,18 +12,17 @@
|
||||
#include "ggml-common.h"
|
||||
#include "htp-ctx.h"
|
||||
#include "htp-ops.h"
|
||||
#include "htp-ops.h"
|
||||
#include "htp-tensor.h"
|
||||
#include "hvx-utils.h"
|
||||
#include "hvx-quant.h"
|
||||
#include "get-rows-ops.h"
|
||||
#include "work-queue.h"
|
||||
|
||||
struct get_rows_context {
|
||||
struct htp_ops_context * octx;
|
||||
uint32_t tasks_per_thread;
|
||||
uint32_t total_tasks;
|
||||
uint32_t chunks_per_row;
|
||||
uint32_t chunk_size;
|
||||
struct fastdiv_values get_rows_div_ne10;
|
||||
struct fastdiv_values get_rows_div_ne10_ne11;
|
||||
struct fastdiv_values get_rows_div_chunks_per_row;
|
||||
const struct htp_get_rows_kernel_params * kparams;
|
||||
struct htp_get_rows_vtcm_layout vtcm_layout;
|
||||
uint8_t * vtcm_base;
|
||||
};
|
||||
|
||||
#define get_rows_preamble \
|
||||
@@ -56,102 +55,161 @@ struct get_rows_context {
|
||||
\
|
||||
const uint32_t nr = ne10 * ne11 * ne12;
|
||||
|
||||
static void get_rows_thread_f32_f32_dma(unsigned int nth, unsigned int ith, void *data) {
|
||||
struct get_rows_context * grctx = (struct get_rows_context *)data;
|
||||
struct htp_ops_context * octx = grctx->octx;
|
||||
get_rows_preamble;
|
||||
|
||||
uint64_t qt = HAP_perf_get_qtimer_count();
|
||||
|
||||
const uint32_t dr = grctx->tasks_per_thread;
|
||||
const uint32_t ir0 = dr * ith;
|
||||
if (ir0 >= grctx->total_tasks) {
|
||||
return;
|
||||
}
|
||||
const uint32_t ir1 = MIN(ir0 + dr, grctx->total_tasks);
|
||||
|
||||
const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32);
|
||||
|
||||
dma_queue * dma_queue = octx->ctx->dma[ith];
|
||||
for (uint32_t i = ir0; i < ir1; ++i) {
|
||||
const uint32_t i12 = fastdiv(i, &grctx->get_rows_div_ne10_ne11);
|
||||
const uint32_t rem = i - i12 * ne11 * ne10;
|
||||
const uint32_t i11 = fastdiv(rem, &grctx->get_rows_div_ne10);
|
||||
const uint32_t i10 = rem - i11 * ne10;
|
||||
|
||||
const uintptr_t src1_addr = octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12;
|
||||
uint32_t i01 = is_i32 ? *(int32_t *)src1_addr : *(int64_t *)src1_addr;
|
||||
|
||||
if (i01 >= ne01) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const uintptr_t src0_ptr = octx->src[0]->data + i01*nb01 + i11*nb02 + i12*nb03;
|
||||
const uintptr_t dst_ptr = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3;
|
||||
|
||||
while (!dma_queue_push(dma_queue, dma_make_ptr((void *)dst_ptr, (const void *)src0_ptr), nb1, nb01, ne00 * sizeof(float), 1)) {
|
||||
dma_queue_pop(dma_queue);
|
||||
}
|
||||
}
|
||||
dma_queue_flush(dma_queue);
|
||||
|
||||
qt = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - qt);
|
||||
FARF(HIGH, "get-rows-f32-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth,
|
||||
ne00, ne01, ne02, ne03, ir0, ir1, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, (unsigned) qt);
|
||||
#define GET_ROWS_THREAD_ST_FN(IDX_TYPE) \
|
||||
static void get_rows_thread_st_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \
|
||||
struct get_rows_context * grctx = (struct get_rows_context *)data; \
|
||||
struct htp_ops_context * octx = grctx->octx; \
|
||||
const struct htp_get_rows_kernel_params * kparams = grctx->kparams; \
|
||||
get_rows_preamble; \
|
||||
const uint32_t dr = kparams->tasks_per_thread; \
|
||||
const uint32_t ir0 = dr * ith; \
|
||||
if (ir0 >= kparams->total_tasks) { \
|
||||
return; \
|
||||
} \
|
||||
const uint32_t ir1 = MIN(ir0 + dr, kparams->total_tasks); \
|
||||
const uint32_t row_size_bytes = htp_tensor_get_row_size(octx->src[0]->type, ne00); \
|
||||
dma_queue * dma_queue = octx->ctx->dma[ith]; \
|
||||
for (uint32_t i = ir0; i < ir1; ++i) { \
|
||||
const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11); \
|
||||
const uint32_t rem = i - i12 * ne11 * ne10; \
|
||||
const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); \
|
||||
const uint32_t i10 = rem - i11 * ne10; \
|
||||
const IDX_TYPE * src1_ptr = (const IDX_TYPE *)(octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12); \
|
||||
const uint32_t i01 = (uint32_t)*src1_ptr; \
|
||||
assert(i01 < ne01); \
|
||||
const uint32_t q02 = fastdiv(i11, &kparams->div_ne02); \
|
||||
const uint32_t i02 = i11 - q02 * ne02; \
|
||||
const uint32_t q03 = fastdiv(i12, &kparams->div_ne03); \
|
||||
const uint32_t i03 = i12 - q03 * ne03; \
|
||||
const uintptr_t src0_ptr = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03; \
|
||||
const uintptr_t dst_ptr = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3; \
|
||||
while (!dma_queue_push(dma_queue, dma_make_ptr((void *)dst_ptr, (const void *)src0_ptr), nb1, nb01, \
|
||||
row_size_bytes, 1)) { \
|
||||
dma_queue_pop(dma_queue); \
|
||||
} \
|
||||
} \
|
||||
dma_queue_flush(dma_queue); \
|
||||
}
|
||||
|
||||
static void get_rows_thread_f32_f32_hvx(unsigned int nth, unsigned int ith, void *data) {
|
||||
struct get_rows_context * grctx = (struct get_rows_context *)data;
|
||||
struct htp_ops_context * octx = grctx->octx;
|
||||
get_rows_preamble;
|
||||
GET_ROWS_THREAD_ST_FN(int32_t)
|
||||
GET_ROWS_THREAD_ST_FN(int64_t)
|
||||
|
||||
uint64_t qt = HAP_perf_get_qtimer_count();
|
||||
|
||||
const uint32_t dr = grctx->tasks_per_thread;
|
||||
const uint32_t ir0 = dr * ith;
|
||||
if (ir0 >= grctx->total_tasks) {
|
||||
return;
|
||||
}
|
||||
const uint32_t ir1 = MIN(ir0 + dr, grctx->total_tasks);
|
||||
|
||||
const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32);
|
||||
|
||||
const uint32_t chunks_per_row = grctx->chunks_per_row;
|
||||
const uint32_t chunk_size = grctx->chunk_size;
|
||||
for (uint32_t i = ir0; i < ir1; ++i) {
|
||||
const uint32_t row_idx = fastdiv(i, &grctx->get_rows_div_chunks_per_row);
|
||||
const uint32_t chunk_idx = i - row_idx * chunks_per_row;
|
||||
|
||||
const uint32_t i12 = fastdiv(row_idx, &grctx->get_rows_div_ne10_ne11);
|
||||
const uint32_t rem = row_idx - i12 * ne11 * ne10;
|
||||
const uint32_t i11 = fastdiv(rem, &grctx->get_rows_div_ne10);
|
||||
const uint32_t i10 = rem - i11 * ne10;
|
||||
|
||||
const uintptr_t src1_addr = octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12;
|
||||
uint32_t i01 = is_i32 ? *(int32_t *)src1_addr : *(int64_t *)src1_addr;
|
||||
|
||||
if (i01 >= ne01) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const uint32_t offset = chunk_idx * chunk_size;
|
||||
if (offset < ne00) {
|
||||
const uint32_t copy_size = MIN(chunk_size, ne00 - offset);
|
||||
const uintptr_t src0_ptr = octx->src[0]->data + i01*nb01 + i11*nb02 + i12*nb03 + offset * sizeof(float);
|
||||
const uintptr_t dst_ptr = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3 + offset * sizeof(float);
|
||||
hvx_copy_f32_uu((uint8_t *)dst_ptr, (const uint8_t *)src0_ptr, copy_size);
|
||||
}
|
||||
}
|
||||
|
||||
qt = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - qt);
|
||||
FARF(HIGH, "get-rows-f32-f32-hvx %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth,
|
||||
ne00, ne01, ne02, ne03, ir0, ir1, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, (unsigned) qt);
|
||||
#define GET_ROWS_THREAD_DT_FN(TYPE_NAME, SRC0_SIZE_EXPR, IDX_TYPE, COMPUTE_EXPR) \
|
||||
static void get_rows_thread_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \
|
||||
struct get_rows_context * grctx = (struct get_rows_context *)data; \
|
||||
struct htp_ops_context * octx = grctx->octx; \
|
||||
const struct htp_get_rows_kernel_params * kparams = grctx->kparams; \
|
||||
get_rows_preamble; \
|
||||
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
|
||||
const uint32_t dr = kparams->tasks_per_thread; \
|
||||
const uint32_t ir0 = dr * ith; \
|
||||
if (ir0 >= kparams->total_tasks) { \
|
||||
return; \
|
||||
} \
|
||||
const uint32_t ir1 = MIN(ir0 + dr, kparams->total_tasks); \
|
||||
const uint32_t chunks_per_row = kparams->chunks_per_row; \
|
||||
const uint32_t chunk_size = kparams->chunk_size; \
|
||||
dma_queue * dma_queue = octx->ctx->dma[ith]; \
|
||||
const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout; \
|
||||
uint8_t * vtcm_src0 = grctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread; \
|
||||
uint8_t * vtcm_dst = grctx->vtcm_base + vtcm_layout->off_dst + ith * vtcm_layout->dst_bytes_per_thread; \
|
||||
for (uint32_t step = 0, spad_idx = 0; step < ir1 - ir0 && spad_idx < 2; ++step, spad_idx++) { \
|
||||
const uint32_t i = ir0 + step; \
|
||||
const uint32_t row_idx = fastdiv(i, &kparams->div_chunks_per_row); \
|
||||
const uint32_t chunk_idx = i - row_idx * chunks_per_row; \
|
||||
const uint32_t i12 = fastdiv(row_idx, &kparams->div_ne10_ne11); \
|
||||
const uint32_t rem = row_idx - i12 * ne11 * ne10; \
|
||||
const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); \
|
||||
const uint32_t i10 = rem - i11 * ne10; \
|
||||
const IDX_TYPE * src1_ptr = (const IDX_TYPE *)(octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12); \
|
||||
const uint32_t i01 = (uint32_t)*src1_ptr; \
|
||||
assert(i01 < ne01); \
|
||||
const uint32_t q02 = fastdiv(i11, &kparams->div_ne02); \
|
||||
const uint32_t i02 = i11 - q02 * ne02; \
|
||||
const uint32_t q03 = fastdiv(i12, &kparams->div_ne03); \
|
||||
const uint32_t i03 = i12 - q03 * ne03; \
|
||||
const uint32_t offset = chunk_idx * chunk_size; \
|
||||
const uint32_t cur_elems = (offset < ne00) ? MIN(chunk_size, ne00 - offset) : 0; \
|
||||
const uint32_t cur_src0_bytes = SRC0_SIZE_EXPR(cur_elems); \
|
||||
const uint32_t cur_dst_bytes = cur_elems * sizeof(float); \
|
||||
const uintptr_t src0_ptr = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03 + SRC0_SIZE_EXPR(offset); \
|
||||
dma_queue_push(dma_queue, \
|
||||
dma_make_ptr((void *)(uintptr_t)octx->dst->data, \
|
||||
vtcm_dst + spad_idx * vtcm_layout->dst_spad_half_size), \
|
||||
cur_dst_bytes, vtcm_layout->dst_spad_half_size, cur_dst_bytes, 0); \
|
||||
dma_queue_push(dma_queue, \
|
||||
dma_make_ptr((void *)(vtcm_src0 + spad_idx * vtcm_layout->src0_spad_half_size), \
|
||||
(const void *)src0_ptr), \
|
||||
vtcm_layout->src0_spad_half_size, cur_src0_bytes, cur_src0_bytes, 1); \
|
||||
} \
|
||||
for (uint32_t step = 0; step < ir1 - ir0; ++step) { \
|
||||
const uint32_t i = ir0 + step; \
|
||||
void * dst_spad = (void *) dma_queue_pop(dma_queue).src; \
|
||||
void * src_spad = (void *) dma_queue_pop(dma_queue).dst; \
|
||||
const uint32_t row_idx = fastdiv(i, &kparams->div_chunks_per_row); \
|
||||
const uint32_t chunk_idx = i - row_idx * chunks_per_row; \
|
||||
const uint32_t i12 = fastdiv(row_idx, &kparams->div_ne10_ne11); \
|
||||
const uint32_t rem = row_idx - i12 * ne11 * ne10; \
|
||||
const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); \
|
||||
const uint32_t i10 = rem - i11 * ne10; \
|
||||
const uint32_t offset = chunk_idx * chunk_size; \
|
||||
const uint32_t cur_elems = (offset < ne00) ? MIN(chunk_size, ne00 - offset) : 0; \
|
||||
const uint32_t cur_dst_bytes = cur_elems * sizeof(float); \
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, i); \
|
||||
COMPUTE_EXPR; \
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, i); \
|
||||
const uintptr_t dst_ptr = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3 + offset * sizeof(float); \
|
||||
dma_queue_push(dma_queue, \
|
||||
dma_make_ptr((void *)dst_ptr, (const void *)dst_spad), \
|
||||
cur_dst_bytes, vtcm_layout->dst_spad_half_size, cur_dst_bytes, 1); \
|
||||
const uint32_t next_step = step + 2; \
|
||||
if (next_step < ir1 - ir0) { \
|
||||
const uint32_t pi = ir0 + next_step; \
|
||||
const uint32_t prow_idx = fastdiv(pi, &kparams->div_chunks_per_row); \
|
||||
const uint32_t pchunk_idx = pi - prow_idx * chunks_per_row; \
|
||||
const uint32_t pi12 = fastdiv(prow_idx, &kparams->div_ne10_ne11); \
|
||||
const uint32_t prem = prow_idx - pi12 * ne11 * ne10; \
|
||||
const uint32_t pi11 = fastdiv(prem, &kparams->div_ne10); \
|
||||
const uint32_t pi10 = prem - pi11 * ne10; \
|
||||
const IDX_TYPE * psrc1_ptr = (const IDX_TYPE *)(octx->src[1]->data + pi10*nb10 + pi11*nb11 + pi12*nb12); \
|
||||
const uint32_t pi01 = (uint32_t)*psrc1_ptr; \
|
||||
assert(pi01 < ne01); \
|
||||
const uint32_t pq02 = fastdiv(pi11, &kparams->div_ne02); \
|
||||
const uint32_t pi02 = pi11 - pq02 * ne02; \
|
||||
const uint32_t pq03 = fastdiv(pi12, &kparams->div_ne03); \
|
||||
const uint32_t pi03 = pi12 - pq03 * ne03; \
|
||||
const uint32_t poffset = pchunk_idx * chunk_size; \
|
||||
const uint32_t pcur_elems = (poffset < ne00) ? MIN(chunk_size, ne00 - poffset) : 0; \
|
||||
const uint32_t pcur_src0_bytes = SRC0_SIZE_EXPR(pcur_elems); \
|
||||
const uintptr_t psrc0_ptr = \
|
||||
octx->src[0]->data + pi01*nb01 + pi02*nb02 + pi03*nb03 + SRC0_SIZE_EXPR(poffset); \
|
||||
dma_queue_push(dma_queue, \
|
||||
dma_make_ptr((void *)src_spad, (const void *)psrc0_ptr), \
|
||||
vtcm_layout->src0_spad_half_size, pcur_src0_bytes, pcur_src0_bytes, 1); \
|
||||
} \
|
||||
} \
|
||||
dma_queue_flush(dma_queue); \
|
||||
}
|
||||
|
||||
#define F32_BYTES(n) ((n) * sizeof(float))
|
||||
#define F16_BYTES(n) ((n) * sizeof(__fp16))
|
||||
#define Q8_0_BYTES(n) (((n) / 32) * sizeof(block_q8_0))
|
||||
|
||||
GET_ROWS_THREAD_DT_FN(f32, F32_BYTES, int32_t, { if (cur_elems > 0) hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, cur_elems); })
|
||||
GET_ROWS_THREAD_DT_FN(f32, F32_BYTES, int64_t, { if (cur_elems > 0) hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, cur_elems); })
|
||||
|
||||
GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int32_t, { hvx_dequantize_row_f16_f32((float *)dst_spad, src_spad, ne00); })
|
||||
GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int64_t, { hvx_dequantize_row_f16_f32((float *)dst_spad, src_spad, ne00); })
|
||||
|
||||
GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int32_t, { hvx_dequantize_row_q8_0_f32((float *)dst_spad, src_spad, ne00); })
|
||||
GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int64_t, { hvx_dequantize_row_q8_0_f32((float *)dst_spad, src_spad, ne00); })
|
||||
|
||||
int op_get_rows(struct htp_ops_context * octx) {
|
||||
get_rows_preamble;
|
||||
const struct htp_get_rows_kernel_params * kparams = (const struct htp_get_rows_kernel_params *) octx->kernel_params;
|
||||
|
||||
if (octx->src[0]->type != HTP_TYPE_F32) {
|
||||
if (octx->src[0]->type != HTP_TYPE_F32 &&
|
||||
octx->src[0]->type != HTP_TYPE_F16 &&
|
||||
octx->src[0]->type != HTP_TYPE_Q8_0) {
|
||||
return HTP_STATUS_NO_SUPPORT;
|
||||
}
|
||||
|
||||
@@ -167,52 +225,28 @@ int op_get_rows(struct htp_ops_context * octx) {
|
||||
return HTP_STATUS_OK;
|
||||
}
|
||||
|
||||
const uint32_t nb00 = octx->src[0]->nb[0];
|
||||
const uint32_t nb0 = octx->dst->nb[0];
|
||||
|
||||
const bool can_use_dma = (nb00 == sizeof(float)) && (nb0 == sizeof(float));
|
||||
const bool use_dma = can_use_dma && (ne00 >= 2048);
|
||||
|
||||
struct get_rows_context grctx;
|
||||
grctx.octx = octx;
|
||||
grctx.get_rows_div_ne10 = init_fastdiv_values(octx->src[1]->ne[0]);
|
||||
grctx.get_rows_div_ne10_ne11 = init_fastdiv_values(octx->src[1]->ne[0] * octx->src[1]->ne[1]);
|
||||
grctx.kparams = kparams;
|
||||
grctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base;
|
||||
|
||||
if (use_dma) {
|
||||
grctx.chunks_per_row = 1;
|
||||
grctx.chunk_size = ne00;
|
||||
grctx.total_tasks = nr;
|
||||
grctx.get_rows_div_chunks_per_row = init_fastdiv_values(1);
|
||||
const uint32_t ne00 = octx->src[0]->ne[0];
|
||||
htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, octx->src[0]->type, ne00, kparams->n_threads);
|
||||
|
||||
const uint32_t n_threads = MIN(nr, octx->n_threads);
|
||||
grctx.tasks_per_thread = (nr + n_threads - 1) / n_threads;
|
||||
const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32);
|
||||
|
||||
worker_pool_run_func(octx->ctx->worker_pool, get_rows_thread_f32_f32_dma, &grctx, n_threads);
|
||||
work_queue_func_t q_func = NULL;
|
||||
if (kparams->use_dma) {
|
||||
q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_st_int32_t : get_rows_thread_st_int64_t);
|
||||
} else {
|
||||
uint32_t chunks_per_row = 1;
|
||||
uint32_t chunk_size = ne00;
|
||||
uint32_t total_tasks = nr;
|
||||
|
||||
if (nr < octx->n_threads) {
|
||||
const uint32_t min_chunk_size = 1024;
|
||||
uint32_t max_chunks = ne00 / min_chunk_size;
|
||||
if (max_chunks == 0) {
|
||||
max_chunks = 1;
|
||||
}
|
||||
chunks_per_row = MIN((octx->n_threads + nr - 1) / nr, max_chunks);
|
||||
chunk_size = (ne00 + chunks_per_row - 1) / chunks_per_row;
|
||||
total_tasks = nr * chunks_per_row;
|
||||
switch (octx->src[0]->type) {
|
||||
case HTP_TYPE_F32: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_f32_int32_t : get_rows_thread_f32_int64_t); break;
|
||||
case HTP_TYPE_F16: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_f16_int32_t : get_rows_thread_f16_int64_t); break;
|
||||
case HTP_TYPE_Q8_0: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_q8_0_int32_t : get_rows_thread_q8_0_int64_t); break;
|
||||
default: return HTP_STATUS_NO_SUPPORT;
|
||||
}
|
||||
|
||||
grctx.chunks_per_row = chunks_per_row;
|
||||
grctx.chunk_size = chunk_size;
|
||||
grctx.total_tasks = total_tasks;
|
||||
grctx.get_rows_div_chunks_per_row = init_fastdiv_values(chunks_per_row);
|
||||
|
||||
const uint32_t n_threads = MIN(total_tasks, octx->n_threads);
|
||||
grctx.tasks_per_thread = (total_tasks + n_threads - 1) / n_threads;
|
||||
|
||||
worker_pool_run_func(octx->ctx->worker_pool, get_rows_thread_f32_f32_hvx, &grctx, n_threads);
|
||||
}
|
||||
|
||||
work_queue_run(octx->ctx->work_queue, q_func, &grctx, kparams->n_threads);
|
||||
return HTP_STATUS_OK;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user