Max Krasnyansky
b2dd28a3b6
* hex-mm: fix artificial limit in the solver that restricted number of act-prep threads * hex-mm: fix warning * hex-prof: do not apply --top to the timeline report * hmx-mm: add suport for tiled act-processing to better distribute hvx work * hex-l2: add tracing for l2flush events * workqueue: redo the legacy workpool api to match hmx-queue and dma-queue * hmx-mm: fix f32 activation buffer alignmnet for nhvx=5,6,7 * hex-work: minor cleanup for work-queue apis * hex-work: further cleanup of the work-queue api * hex-l2: optimize l2flushes at the opbatch level * hex-work: remove unused mask * hex-work: no need to drop hvx ctx in the work-queue * hex-work: add explicit wakeup/suspend and make threads spin * hex-bufs: mark any non-weight tensor as compute * hex-dma: dma-queue support for alias queues and cached dma * hex-l2: track tensor aliases and delay or skip flushes as much as possible * hex-l2: simplify tensor alias handling * hex-l2: handle overlapping views as a circular list of aliases * hex-tens: add flags helper * hex-l2: add helper for marking tensors clearn/dirty * hex-l2: mark binary and rope outputs as l2-clean and keep the rest as is for now * hex-l2: proper support for handling all tensor overlap scenarios * hex-trace: instrument matmul init code and cleanup trace checks * hex-thread: introduce dedicated main thread with explicit stack and priority * hex-l2: track dirty state as bitmap and introduce threaded flush * hex-trace: remove redundant checks for ctx != null * hex-l2: allocate entire context as one buffer and l2fetch it after big flushes * hex-l2: disable tensor clearing in binary and rope for now seems to cause issues with fusion * hmx-mm: update act proc to use fastdivs and fix DMA overflow * hmx-mm: make MUL_MAT_ID kernels robust to multi-chunk cases (start_row>0) * hex-queue: remove obsolete queue interfaces and flush hmx-queue at the end of the op-batch * hex-queue: dont use early wakeup for small op-batches * hex-tensors: properly cap max_tensors in op-batches and dirty_map * hex-l2: make sure threaded l2flush does proper rounding * hex-l2: factor out htp_tensor_flush for reuse (if needed) * hex-l2: optimize tensor flushes by coalescing flush-all * hex-l2: optimize multi-threaded flush * hex-drv: futureproof version checks * hexagon: fix errors and warnings on windows * hex-main: update main thread to only use dspqueue_read, dspqueue_peek is not available on some platforms * hex-main: add fallback mode for dspqueue with callbacks * hex-main: introduce fallback mode for using dspqueue callbacks for full op processing * hex-main: remove early wakeup, not helping and seems to cause some errors with certain batch sizes * hex-l2: make sure to use invalidate version of flushall * hex-l2: dont try to trace early l2flush at the start of op-batch * hex-main: remove offset_ctx that must be zero anyway * hex-hmx: fix hmx_queue_depth to use idx_write - idx_read * hex-hmx: use atomic_load for idx_read/write * hex-main: add static assert to make sure n_threads are aligned
139 lines
4.1 KiB
C
139 lines
4.1 KiB
C
#ifndef HTP_CTX_H
|
|
#define HTP_CTX_H
|
|
|
|
#include "hex-dma.h"
|
|
#include "hmx-queue.h"
|
|
#include "htp-ops.h"
|
|
#include "hex-profile.h"
|
|
#include "work-queue.h"
|
|
#include "hex-fastdiv.h"
|
|
|
|
#include <assert.h>
|
|
#include <dspqueue.h>
|
|
#include <stdatomic.h>
|
|
#include <stdint.h>
|
|
#include <stdbool.h>
|
|
|
|
#ifndef HTP_MAX_NTHREADS
|
|
#define HTP_MAX_NTHREADS 10
|
|
#endif
|
|
#define HTP_MAX_MMAPS 16
|
|
|
|
// Memory mapping
|
|
struct htp_mmap {
|
|
uint64_t size;
|
|
uint64_t base;
|
|
uint32_t fd;
|
|
uint32_t reserved;
|
|
};
|
|
|
|
// Scratchpad state
|
|
struct htp_spad {
|
|
const struct htp_tensor * src; // original src of the data (for reuse)
|
|
uint8_t * data; // pointer to an area in vtcm
|
|
uint32_t stride; // stride used inside this spad
|
|
uint32_t size; // total size
|
|
uint32_t size_per_thread; // size per thread
|
|
};
|
|
|
|
struct htp_context;
|
|
|
|
// Context while processing an Op
|
|
// TODO: fold this into the main context
|
|
struct htp_ops_context {
|
|
struct htp_context * ctx;
|
|
|
|
enum htp_op_code op; // FIXME: rename to opcode
|
|
int32_t op_params[HTP_OP_MAX_PARAMS];
|
|
int32_t kernel_params[HTP_OP_MAX_KERN_PARAMS];
|
|
|
|
const struct htp_tensor * src[HTP_OP_MAX_INPUTS];
|
|
union {
|
|
const struct htp_tensor * dst;
|
|
const struct htp_tensor * dsts[HTP_OP_MAX_OUTPUTS];
|
|
};
|
|
|
|
dma_queue ** src_dma[HTP_OP_MAX_INPUTS];
|
|
dma_queue ** dst_dma[HTP_OP_MAX_OUTPUTS];
|
|
|
|
// TODO convert these to an array
|
|
struct htp_spad src0_spad;
|
|
struct htp_spad src1_spad;
|
|
struct htp_spad src2_spad;
|
|
struct htp_spad src3_spad;
|
|
struct htp_spad dst_spad;
|
|
|
|
uint32_t n_threads;
|
|
uint32_t flags;
|
|
};
|
|
|
|
// Main context for htp DSP backend
|
|
struct htp_context {
|
|
dspqueue_t dsp_queue;
|
|
|
|
struct htp_mmap mmap[HTP_MAX_MMAPS];
|
|
dma_queue_t dma[HTP_MAX_NTHREADS];
|
|
dma_queue_t dma_cached[HTP_MAX_NTHREADS];
|
|
work_queue_t work_queue;
|
|
hmx_queue_t hmx_queue;
|
|
|
|
uint32_t n_threads;
|
|
struct fastdiv_values n_threads_div;
|
|
|
|
int thread_id;
|
|
int thread_prio;
|
|
|
|
bool hmx_enabled;
|
|
bool etm;
|
|
uint32_t profiler;
|
|
struct htp_thread_trace trace[HTP_MAX_NTHREADS + 1];
|
|
|
|
uint8_t * vtcm_base;
|
|
size_t vtcm_size;
|
|
uint32_t vtcm_rctx;
|
|
atomic_bool vtcm_valid;
|
|
atomic_bool vtcm_needs_release;
|
|
|
|
uint64_t max_vmem;
|
|
uint32_t dirty_map[HTP_OP_MAX_TENSORS / 32];
|
|
|
|
// Persistent DDR scratchpad for MUL_MAT_ID mappings
|
|
void * ddr_spad_base;
|
|
size_t ddr_spad_size;
|
|
|
|
struct htp_ops_context octx;
|
|
|
|
qurt_thread_t main_thread;
|
|
void * main_stack;
|
|
atomic_bool killed;
|
|
size_t footprint;
|
|
};
|
|
|
|
int op_matmul(struct htp_ops_context * octx);
|
|
int op_matmul_id(struct htp_ops_context * octx);
|
|
int op_matmul_qkv(struct htp_ops_context * octx);
|
|
int op_matmul_ffn(struct htp_ops_context * octx);
|
|
int op_binary(struct htp_ops_context * octx);
|
|
int op_unary(struct htp_ops_context * octx);
|
|
int op_sum_rows(struct htp_ops_context * octx);
|
|
int op_activations(struct htp_ops_context * octx);
|
|
int op_softmax(struct htp_ops_context * octx);
|
|
int op_add_id(struct htp_ops_context * octx);
|
|
int op_rope(struct htp_ops_context * octx);
|
|
int op_flash_attn_ext(struct htp_ops_context * octx);
|
|
int op_set_rows(struct htp_ops_context * octx);
|
|
int op_get_rows(struct htp_ops_context * octx);
|
|
int op_cpy(struct htp_ops_context * octx);
|
|
int op_repeat(struct htp_ops_context * octx);
|
|
int op_argsort(struct htp_ops_context * octx);
|
|
int op_ssm_conv(struct htp_ops_context * octx);
|
|
int op_cumsum(struct htp_ops_context * octx);
|
|
int op_fill(struct htp_ops_context * octx);
|
|
int op_concat(struct htp_ops_context * octx);
|
|
int op_diag(struct htp_ops_context * octx);
|
|
int op_solve_tri(struct htp_ops_context * octx);
|
|
int op_gated_delta_net(struct htp_ops_context * octx);
|
|
int op_pad(struct htp_ops_context * octx);
|
|
|
|
#endif /* HTP_CTX_H */
|