Max Krasnyansky
192067b72d
* hexagon: use non-host bufs by default and make the backend fully async * hex-hb: remove optional hostbuf support and fix async copy * hex-unary: relax supported unary check * hex-bufs: use same get_alignment for host bufs * snapdragon: bump android_platform to 34 * hex-rows: super hacky get/set rows for q8_0 * hex-get-rows: fix q8_0 * hex-get-rows: supprot for f16 and cleanup for q8_0 * hex-get-rows: generic macros and specialized thread funcs * hex-get-rows: add DMA pipeline, vtcm_layout and kernel params * hex-set-rows: fix q8_0 support, add dma and tracing * hex-tests: override nmse threshold for HTP of Q8_0 quants * hex-fa: add support for Q8_0 with inplace dequantizers * hex-get-rows: simplify type dispatch * hex-rows: simplify GET/SET_ROWS DMA pipeline * hex-async: add events, set/get-tensor-async and rest of the async api support * hex-repack: use slice instead of expert in repack functions * hex-cpy: update event/async-cpy logging * hex-set-rows: optimize smaller tensors * hex-geglu: fix perf regression with larger tensors * hex-get-rows: add missing header * hex-set-rows: add missing header * hex-bufs: ressurect GGML_HEXAGON_HOSTBUF but disable it by default * hexagon: do not reject ops with non-heaxon buffers * hex-get-rows: apply >=32 restriction only for q8_0 * hex-res: bump vtcm acquire timeout to 10 seconds * hex-bufs: add support for cloning buffers between sessions to speed up tensor copies * hex-async: rework event recording and batch flushing and integrate with meta backend * hex-bufs: improved handling of repacked tensors * hex-repack: handle get_tensor_2d offsets * hex-dev: add support for devices with multiple NPUs * hex-sync: add support for sync tokens to synchronize npu devices for async splits * hex-mmap: cleanup mmap calls and add a retry for robustness * hex-sync: add failsafe if sync wait gets stuck * hex-sync: use sync_seq to check for completed events * hex-sync: rotate tokens for extra robustness * hex-devs: add supprot for legacy device names for now * hex-bufs: add support for auto-cloning buffers from diff sessions * hex-fusion: simplify and optimize htp-opnode fusion handling * hex-sync: override opnode name so that it shows up in the profiles * hex-trace: update scripts to handle multiple devices * hex-sync: bump the size of the opbatch queue and number of sync tokens * hex-cpy-sync: do not explicitly flush opbatches in cpy_tensor_async and add support for cpy-dma * hex-sync: add graph-flush threshold to avoid single op batches * hex-sync: add sync_peer so that we can flush peers we depend on during cross-device ops * hex-bufs: introduce tensor->extra and shadow_bufs for repacking * hex-l2: flush tiny tensors inline * hex-sync: use explicit l2flush for sync tokens * hex-extra: track weight flags via tensor extra * hex-fence: rename sync to fence * hex-repack: proper handling of set-tensor-2d in the shadow_buf * hex-trace: remove obsolete opstage mask that we used for profiling * hex-env: remove obsolete use_hmx variable * hexagon: new unified run.py and build.py and updated docs * snapdragon: update run script to auto-escapt test-backend-op -p argument * hex-scripts: fix trailing spaces * hex-scripts: fix flake8 warnings * snapdragon: cleanup dst lib/bin dirs before copying new build * hex-ops: add support for allreduce * hex-ar: improved allreduce with dma pipeline * hex-ar: align macros * hex-ar: consistent use of fence_seq * hex-ar: add AR_SELECT env var to select ALLREDUCE kernel or fallback * hex-ar: add proper synchronize handling for ALLREDUCE * hex-opbatch: looks like we now just rely on backend.synchronise to flush the batches, no need to flush them by threshold * hex-ar: bump block size to improve dma efficiency * hex-ar: fused ALLREDUCE+ADD * hex-ar: cleaner fence buffer management * hex-ar: futher allreduce tweaking to remove race conditions * hex-ar: add simple solver and remove non-dma kernels * hex-ar: add row-broadcast to fuse with bias ADD * hex-fence: pass seq numbers via op_params * hex-ar: allow for both entry/exit seq for completing entry wait * hex-ar: align macros * hex-ar: do not refetch broadcast row * hex-fusion: move all fusion into opbatch::add_op for consistency with ALLREDUCE and things * hex-fusion: fix incorrect MUL_MAT reordering * hex-mm: make fused 2x and 3x matmuls more generic * hex-fusion: move tensor fusion tagging to graph_compute * hexagon: make sure to copy tensor->extra by value * hex-get-rows: fix offset calc with row-chunking * hex-repack: get_tensor_2d fixes for non-zero offsets * snapdragon: make profile/trace scripts more robust and donot mix stdout/stderr by default * hex-devices: use legacy device nameing by default to ease the transition * hex-devices: hardcode CDSP domain IDs for current devices for now * hex-optrace: improve multi-NPU timestamp alignment and overall handling of cycle values * hex-optrace: more robust handling of the fence events
351 lines
21 KiB
C
351 lines
21 KiB
C
#pragma clang diagnostic ignored "-Wunused-variable"
|
|
#pragma clang diagnostic ignored "-Wunused-function"
|
|
#pragma clang diagnostic ignored "-Wunused-but-set-variable"
|
|
|
|
#include <HAP_farf.h>
|
|
#include <HAP_perf.h>
|
|
#include <qurt_memory.h>
|
|
|
|
#include <math.h>
|
|
#include <string.h>
|
|
|
|
#define GGML_COMMON_DECL_C
|
|
#include "ggml-common.h"
|
|
#include "htp-ctx.h"
|
|
#include "htp-ops.h"
|
|
#include "htp-ops.h"
|
|
#include "hvx-utils.h"
|
|
#include "htp-tensor.h"
|
|
|
|
struct htp_copy_context {
|
|
struct htp_ops_context * octx;
|
|
|
|
uint32_t src0_type_size;
|
|
uint32_t src0_block_size;
|
|
|
|
uint32_t dst_type_size;
|
|
uint32_t dst_block_size;
|
|
|
|
uint32_t src0_blocks_per_row;
|
|
uint32_t dst_blocks_per_row;
|
|
|
|
uint32_t src0_nrows_per_thread;
|
|
};
|
|
|
|
#define cpy_preamble \
|
|
const struct htp_tensor *src0 = octx->src[0]; \
|
|
const struct htp_tensor *dst = octx->dst; \
|
|
\
|
|
const uint32_t ne00 = src0->ne[0]; \
|
|
const uint32_t ne01 = src0->ne[1]; \
|
|
const uint32_t ne02 = src0->ne[2]; \
|
|
const uint32_t ne03 = src0->ne[3]; \
|
|
\
|
|
const uint32_t nb00 = src0->nb[0]; \
|
|
const uint32_t nb01 = src0->nb[1]; \
|
|
const uint32_t nb02 = src0->nb[2]; \
|
|
const uint32_t nb03 = src0->nb[3]; \
|
|
\
|
|
const uint32_t ne0 = dst->ne[0]; \
|
|
const uint32_t ne1 = dst->ne[1]; \
|
|
const uint32_t ne2 = dst->ne[2]; \
|
|
const uint32_t ne3 = dst->ne[3]; \
|
|
\
|
|
const uint32_t nb0 = dst->nb[0]; \
|
|
const uint32_t nb1 = dst->nb[1]; \
|
|
const uint32_t nb2 = dst->nb[2]; \
|
|
const uint32_t nb3 = dst->nb[3]; \
|
|
\
|
|
const uint32_t nr = ne01;
|
|
|
|
#define DEFINE_CPY_SAMESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \
|
|
static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \
|
|
struct htp_copy_context * ct = (struct htp_copy_context *) data; \
|
|
struct htp_ops_context * octx = ct->octx; \
|
|
cpy_preamble; \
|
|
const uint32_t dr = ct->src0_nrows_per_thread; \
|
|
const uint32_t ir0 = dr * ith; \
|
|
const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr; \
|
|
if (ir0 >= nr) return; \
|
|
for (uint32_t i03 = 0; i03 < ne03; i03++) { \
|
|
for (uint32_t i02 = 0; i02 < ne02; i02++) { \
|
|
_Pragma("unroll(4)") \
|
|
for (uint32_t i01 = ir0; i01 < ir1; i01++) { \
|
|
uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3; \
|
|
uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03; \
|
|
hex_l2fetch(src0_ptr, ne00 * ELEM_SIZE, nb01, 2); \
|
|
hvx_copy_uu(dst_ptr, src0_ptr, ne00, ELEM_SIZE); \
|
|
} \
|
|
} \
|
|
} \
|
|
}
|
|
|
|
DEFINE_CPY_SAMESHAPE(f32, float, 4)
|
|
DEFINE_CPY_SAMESHAPE(f16, __fp16, 2)
|
|
|
|
#define DEFINE_CPY_RESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \
|
|
static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void * data) { \
|
|
struct htp_copy_context * ct = (struct htp_copy_context *) data; \
|
|
struct htp_ops_context * octx = ct->octx; \
|
|
cpy_preamble; \
|
|
const uint32_t dr = ct->src0_nrows_per_thread; \
|
|
const uint32_t ir0 = dr * ith; \
|
|
const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr; \
|
|
if (ir0 >= nr) return; \
|
|
const bool src0_contig = (nb00 == ELEM_SIZE) && \
|
|
(nb01 == ne00 * nb00) && \
|
|
(nb02 == ne01 * nb01) && \
|
|
(nb03 == ne02 * nb02); \
|
|
const bool dst_contig = (nb0 == ELEM_SIZE) && \
|
|
(nb1 == ne0 * nb0) && \
|
|
(nb2 == ne1 * nb1) && \
|
|
(nb3 == ne2 * nb2); \
|
|
if (src0_contig && dst_contig) { \
|
|
for (int64_t i03 = 0; i03 < ne03; i03++) { \
|
|
for (int64_t i02 = 0; i02 < ne02; i02++) { \
|
|
uint8_t * src_ptr = (uint8_t *) src0->data + i03*nb03 + i02*nb02 + ir0*nb01; \
|
|
uint32_t flat = ((i03*ne02 + i02)*ne01 + ir0) * ne00; \
|
|
uint8_t * dst_ptr = (uint8_t *) dst->data + flat * ELEM_SIZE; \
|
|
hvx_copy_uu(dst_ptr, src_ptr, (ir1 - ir0) * ne00, ELEM_SIZE); \
|
|
} \
|
|
} \
|
|
return; \
|
|
} \
|
|
const bool reshape_flat_fast = (ne03 == 1 && ne2 == 1 && ne3 == 1) && \
|
|
(ne0 == ne00 * ne01) && (ne1 == ne02) && \
|
|
(nb00 == ELEM_SIZE) && (nb0 == ELEM_SIZE); \
|
|
if (reshape_flat_fast) { \
|
|
for (uint32_t i02 = 0; i02 < ne02; i02++) { \
|
|
for (uint32_t i01 = ir0; i01 < ir1; i01++) { \
|
|
uint8_t * src0_ptr = (uint8_t *) src0->data + i01 * nb01 + i02 * nb02; \
|
|
uint8_t * dst_ptr = (uint8_t *) dst->data + i01 * ne00 * ELEM_SIZE + i02 * nb1; \
|
|
hvx_copy_uu(dst_ptr, src0_ptr, ne00, ELEM_SIZE); \
|
|
} \
|
|
} \
|
|
return; \
|
|
} \
|
|
int64_t k10 = 0; \
|
|
int64_t i11 = 0; \
|
|
int64_t i12 = 0; \
|
|
int64_t i13 = 0; \
|
|
const int64_t nk00 = ct->src0_blocks_per_row; \
|
|
const int64_t nk0 = ct->dst_blocks_per_row; \
|
|
for (int64_t i03 = 0; i03 < ne03; i03++) { \
|
|
for (int64_t i02 = 0; i02 < ne02; i02++) { \
|
|
k10 += nk00 * ir0; \
|
|
while (k10 >= nk0) { \
|
|
k10 -= nk0; \
|
|
if (++i11 == ne1) { \
|
|
i11 = 0; \
|
|
if (++i12 == ne2) { \
|
|
i12 = 0; \
|
|
if (++i13 == ne3) { \
|
|
i13 = 0; \
|
|
} \
|
|
} \
|
|
} \
|
|
} \
|
|
for (int64_t i01 = ir0; i01 < ir1; i01++) { \
|
|
for (int64_t k00 = 0; k00 < nk00; k00++) { \
|
|
const char * src0_ptr = ((char *) src0->data + k00*nb00 + i01*nb01 + i02*nb02 + i03*nb03); \
|
|
char * dst_ptr = ((char *) dst->data + k10*nb0 + i11*nb1 + i12*nb2 + i13*nb3); \
|
|
memcpy(dst_ptr, src0_ptr, ELEM_SIZE); \
|
|
if (++k10 == nk0) { \
|
|
k10 = 0; \
|
|
if (++i11 == ne1) { \
|
|
i11 = 0; \
|
|
if (++i12 == ne2) { \
|
|
i12 = 0; \
|
|
if (++i13 == ne3) { \
|
|
i13 = 0; \
|
|
} \
|
|
} \
|
|
} \
|
|
} \
|
|
} \
|
|
} \
|
|
k10 += nk00 * (ne01 - ir1); \
|
|
while (k10 >= nk0) { \
|
|
k10 -= nk0; \
|
|
if (++i11 == ne1) { \
|
|
i11 = 0; \
|
|
if (++i12 == ne2) { \
|
|
i12 = 0; \
|
|
if (++i13 == ne3) { \
|
|
i13 = 0; \
|
|
} \
|
|
} \
|
|
} \
|
|
} \
|
|
} \
|
|
} \
|
|
}
|
|
|
|
DEFINE_CPY_RESHAPE(f32, float, 4)
|
|
DEFINE_CPY_RESHAPE(f16, __fp16, 2)
|
|
|
|
static void cpy_thread_f16_f32_sameshape(unsigned int nth, unsigned int ith, void * data) {
|
|
struct htp_copy_context * ct = (struct htp_copy_context *) data;
|
|
struct htp_ops_context * octx = ct->octx;
|
|
cpy_preamble;
|
|
|
|
// parallelize by src0 rows
|
|
const uint32_t dr = ct->src0_nrows_per_thread;
|
|
const uint32_t ir0 = dr * ith;
|
|
const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr;
|
|
if (ir0 >= nr) return;
|
|
|
|
// copy by rows
|
|
for (uint32_t i03 = 0; i03 < ne03; i03++) {
|
|
for (uint32_t i02 = 0; i02 < ne02; i02++) {
|
|
#pragma unroll(2)
|
|
for (uint32_t i01 = ir0; i01 < ir1; i01++) {
|
|
uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3;
|
|
uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03;
|
|
hex_l2fetch(src0_ptr, ne00 * sizeof(float), nb01, 2);
|
|
hvx_copy_f16_f32_uu(dst_ptr, src0_ptr, ne00);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
static void cpy_thread_f32_f16_sameshape(unsigned int nth, unsigned int ith, void * data) {
|
|
struct htp_copy_context * ct = (struct htp_copy_context *) data;
|
|
struct htp_ops_context * octx = ct->octx;
|
|
cpy_preamble;
|
|
|
|
// parallelize by src0 rows
|
|
const uint32_t dr = ct->src0_nrows_per_thread;
|
|
const uint32_t ir0 = dr * ith;
|
|
const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr;
|
|
if (ir0 >= nr) return;
|
|
|
|
// copy by rows
|
|
for (uint32_t i03 = 0; i03 < ne03; i03++) {
|
|
for (uint32_t i02 = 0; i02 < ne02; i02++) {
|
|
#pragma unroll(2)
|
|
for (uint32_t i01 = ir0; i01 < ir1; i01++) {
|
|
uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3;
|
|
uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03;
|
|
hex_l2fetch(src0_ptr, ne00 * sizeof(__fp16), nb01, 2);
|
|
hvx_copy_f32_f16_uu(dst_ptr, src0_ptr, ne00);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
static inline void cpy_dma_sametype_sameshape(
|
|
struct htp_ops_context * octx,
|
|
const struct htp_tensor * dst,
|
|
const struct htp_tensor * src0,
|
|
uint32_t elem_size,
|
|
uint32_t ne00, uint32_t ne01, uint32_t ne02, uint32_t ne03,
|
|
uint32_t nb01, uint32_t nb02, uint32_t nb03,
|
|
uint32_t nb1, uint32_t nb2, uint32_t nb3
|
|
) {
|
|
const bool contiguous_outer =
|
|
(ne02 == 1 || (nb02 == ne01 * nb01 && nb2 == ne01 * nb1)) &&
|
|
(ne03 == 1 || (nb03 == ne02 * nb02 && nb3 == ne02 * nb2));
|
|
|
|
dma_queue * q = octx->ctx->dma[0];
|
|
|
|
if (contiguous_outer) {
|
|
dma_queue_push(q, dma_make_ptr((void *) dst->data, (const void *) src0->data), nb1, nb01, ne00 * elem_size, ne01 * ne02 * ne03);
|
|
dma_queue_pop(q);
|
|
return;
|
|
}
|
|
|
|
for (uint32_t i03 = 0; i03 < ne03; i03++) {
|
|
for (uint32_t i02 = 0; i02 < ne02; i02++) {
|
|
uint8_t* dst_ptr = (uint8_t*) dst->data + i02*nb2 + i03*nb3;
|
|
uint8_t* src0_ptr = (uint8_t*) src0->data + i02*nb02 + i03*nb03;
|
|
if (!dma_queue_push(q, dma_make_ptr(dst_ptr, src0_ptr), nb1, nb01, ne00 * elem_size, ne01)) {
|
|
dma_queue_flush(q);
|
|
dma_queue_push(q, dma_make_ptr(dst_ptr, src0_ptr), nb1, nb01, ne00 * elem_size, ne01);
|
|
}
|
|
}
|
|
}
|
|
|
|
dma_queue_flush(q);
|
|
}
|
|
|
|
int op_cpy(struct htp_ops_context * octx) {
|
|
cpy_preamble;
|
|
|
|
const uint32_t n_threads = MIN(nr, octx->n_threads);
|
|
|
|
struct htp_copy_context ct;
|
|
ct.octx = octx;
|
|
|
|
switch (src0->type) {
|
|
case HTP_TYPE_F32: ct.src0_type_size = 4; ct.src0_block_size = 1; ct.src0_blocks_per_row = ne00 / 1; break;
|
|
case HTP_TYPE_F16: ct.src0_type_size = 2; ct.src0_block_size = 1; ct.src0_blocks_per_row = ne00 / 1; break;
|
|
default:
|
|
return HTP_STATUS_NO_SUPPORT;
|
|
}
|
|
|
|
switch (dst->type) {
|
|
case HTP_TYPE_F32: ct.dst_type_size = 4; ct.dst_block_size = 1; ct.dst_blocks_per_row = ne0 / 1; break;
|
|
case HTP_TYPE_F16: ct.dst_type_size = 2; ct.dst_block_size = 1; ct.dst_blocks_per_row = ne0 / 1; break;
|
|
default:
|
|
return HTP_STATUS_NO_SUPPORT;
|
|
}
|
|
|
|
if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) {
|
|
return HTP_STATUS_OK;
|
|
}
|
|
|
|
const bool sametype = (src0->type == dst->type);
|
|
const bool transposed = (nb00 > nb01) || (nb0 > nb1);
|
|
const bool sameshape = !transposed && (ne00 == ne0 && ne01 == ne1 && ne02 == ne2 && ne03 == ne3);
|
|
|
|
ct.src0_nrows_per_thread = (nr + n_threads - 1) / n_threads;
|
|
|
|
worker_callback_t copy_fun = NULL;
|
|
bool use_dma = false;
|
|
|
|
if (sametype && sameshape) {
|
|
use_dma = true;
|
|
} else if (sameshape) {
|
|
/**/ if (dst->type == HTP_TYPE_F16 && src0->type == HTP_TYPE_F32)
|
|
copy_fun = cpy_thread_f16_f32_sameshape;
|
|
else if (dst->type == HTP_TYPE_F32 && src0->type == HTP_TYPE_F16)
|
|
copy_fun = cpy_thread_f32_f16_sameshape;
|
|
else
|
|
return HTP_STATUS_NO_SUPPORT;
|
|
} else if (sametype) {
|
|
if (src0->type == HTP_TYPE_F32) {
|
|
copy_fun = cpy_thread_f32_reshape;
|
|
} else {
|
|
copy_fun = cpy_thread_f16_reshape;
|
|
}
|
|
} else {
|
|
return HTP_STATUS_NO_SUPPORT;
|
|
}
|
|
|
|
if (use_dma) {
|
|
cpy_dma_sametype_sameshape(octx, dst, src0, ct.src0_type_size, ne00, ne01, ne02, ne03, nb01, nb02, nb03, nb1, nb2, nb3);
|
|
} else {
|
|
worker_pool_run_func(octx->ctx->worker_pool, copy_fun, &ct, n_threads);
|
|
}
|
|
|
|
const struct htp_tensor *sync = octx->src[1];
|
|
if (sync) {
|
|
if (!use_dma) {
|
|
// htp_tensor_flush_all(octx->ctx, octx->dsts, 1);
|
|
qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE);
|
|
}
|
|
|
|
atomic_uint * sync_fence = (atomic_uint *) sync->data;
|
|
const uint32_t seq = (uint32_t) octx->op_params[0];
|
|
|
|
atomic_store(&sync_fence[0], seq);
|
|
asm volatile ("syncht" : : : "memory");
|
|
Q6_dccleaninva_A((void *) sync_fence);
|
|
|
|
FARF(HIGH, "ggml-hex: sync-release : fence %p seq %u\n", sync_fence, seq);
|
|
}
|
|
|
|
return HTP_STATUS_OK;
|
|
}
|