hexagon: L2 cache handling rework (dirty bit tracking with lazy flushing) and more MUL_MAT updates (#25762)
* hex-mm: fix artificial limit in the solver that restricted number of act-prep threads * hex-mm: fix warning * hex-prof: do not apply --top to the timeline report * hmx-mm: add suport for tiled act-processing to better distribute hvx work * hex-l2: add tracing for l2flush events * workqueue: redo the legacy workpool api to match hmx-queue and dma-queue * hmx-mm: fix f32 activation buffer alignmnet for nhvx=5,6,7 * hex-work: minor cleanup for work-queue apis * hex-work: further cleanup of the work-queue api * hex-l2: optimize l2flushes at the opbatch level * hex-work: remove unused mask * hex-work: no need to drop hvx ctx in the work-queue * hex-work: add explicit wakeup/suspend and make threads spin * hex-bufs: mark any non-weight tensor as compute * hex-dma: dma-queue support for alias queues and cached dma * hex-l2: track tensor aliases and delay or skip flushes as much as possible * hex-l2: simplify tensor alias handling * hex-l2: handle overlapping views as a circular list of aliases * hex-tens: add flags helper * hex-l2: add helper for marking tensors clearn/dirty * hex-l2: mark binary and rope outputs as l2-clean and keep the rest as is for now * hex-l2: proper support for handling all tensor overlap scenarios * hex-trace: instrument matmul init code and cleanup trace checks * hex-thread: introduce dedicated main thread with explicit stack and priority * hex-l2: track dirty state as bitmap and introduce threaded flush * hex-trace: remove redundant checks for ctx != null * hex-l2: allocate entire context as one buffer and l2fetch it after big flushes * hex-l2: disable tensor clearing in binary and rope for now seems to cause issues with fusion * hmx-mm: update act proc to use fastdivs and fix DMA overflow * hmx-mm: make MUL_MAT_ID kernels robust to multi-chunk cases (start_row>0) * hex-queue: remove obsolete queue interfaces and flush hmx-queue at the end of the op-batch * hex-queue: dont use early wakeup for small op-batches * hex-tensors: properly cap max_tensors in op-batches and dirty_map * hex-l2: make sure threaded l2flush does proper rounding * hex-l2: factor out htp_tensor_flush for reuse (if needed) * hex-l2: optimize tensor flushes by coalescing flush-all * hex-l2: optimize multi-threaded flush * hex-drv: futureproof version checks * hexagon: fix errors and warnings on windows * hex-main: update main thread to only use dspqueue_read, dspqueue_peek is not available on some platforms * hex-main: add fallback mode for dspqueue with callbacks * hex-main: introduce fallback mode for using dspqueue callbacks for full op processing * hex-main: remove early wakeup, not helping and seems to cause some errors with certain batch sizes * hex-l2: make sure to use invalidate version of flushall * hex-l2: dont try to trace early l2flush at the start of op-batch * hex-main: remove offset_ctx that must be zero anyway * hex-hmx: fix hmx_queue_depth to use idx_write - idx_read * hex-hmx: use atomic_load for idx_read/write * hex-main: add static assert to make sure n_threads are aligned
This commit is contained in:
Max Krasnyansky
@@ -0,0 +1,204 @@
|
||||
#include "htp-tensor.h"
|
||||
|
||||
#include <qurt.h>
|
||||
#include <qurt_memory.h>
|
||||
|
||||
#include "hex-common.h"
|
||||
#include "hex-utils.h"
|
||||
#include "hex-fastdiv.h"
|
||||
#include "hex-profile.h"
|
||||
#include "htp-ctx.h"
|
||||
#include "work-queue.h"
|
||||
|
||||
struct l2flush_task {
|
||||
struct htp_thread_trace * trace;
|
||||
uint32_t start;
|
||||
uint32_t end;
|
||||
uint32_t chunk_size;
|
||||
uint32_t ti;
|
||||
};
|
||||
|
||||
static void l2flush_thread_worker(unsigned int n, unsigned int i, void * data) {
|
||||
struct l2flush_task * task = (struct l2flush_task *) data;
|
||||
const uint32_t start = task->start;
|
||||
const uint32_t end = task->end;
|
||||
const uint32_t ti = task->ti;
|
||||
const uint32_t chunk_size = task->chunk_size;
|
||||
|
||||
const uint32_t thread_s = start + i * chunk_size;
|
||||
if (thread_s >= end) {
|
||||
return;
|
||||
}
|
||||
uint32_t thread_e = thread_s + chunk_size;
|
||||
if (thread_e > end) {
|
||||
thread_e = end;
|
||||
}
|
||||
|
||||
struct htp_thread_trace * tr = &task->trace[i];
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, ti);
|
||||
hex_l2flush((void *) (uintptr_t) thread_s, thread_e - thread_s);
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, ti);
|
||||
}
|
||||
|
||||
static void flush_all_dcache(struct htp_context * ctx) {
|
||||
struct htp_thread_trace * tr = &ctx->trace[0];
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, 0);
|
||||
qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE);
|
||||
hex_l2fetch_block(ctx, ctx->footprint);
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, 0);
|
||||
bitmap_reset(ctx->dirty_map, HTP_OP_MAX_TENSORS);
|
||||
}
|
||||
|
||||
static void flush_tensor_range(struct htp_context * ctx, const struct htp_tensor * t) {
|
||||
struct htp_thread_trace * tr = &ctx->trace[0];
|
||||
|
||||
if (t->size > HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1) {
|
||||
struct l2flush_task task;
|
||||
task.start = hex_align_down((size_t) t->data, HEX_L2_LINE_SIZE);
|
||||
task.end = hex_align_up((size_t) t->data + t->size, HEX_L2_LINE_SIZE);
|
||||
task.ti = t->ti;
|
||||
task.trace = ctx->trace;
|
||||
|
||||
const uint32_t total_size = task.end - task.start;
|
||||
const uint32_t n_blocks = (total_size + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE;
|
||||
const uint32_t blocks_per_thread = fastdiv(n_blocks + ctx->n_threads - 1, &ctx->n_threads_div);
|
||||
task.chunk_size = blocks_per_thread * HEX_L2_BLOCK_SIZE;
|
||||
|
||||
work_queue_run(ctx->work_queue, l2flush_thread_worker, &task, ctx->n_threads);
|
||||
} else {
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, t->ti);
|
||||
hex_l2flush((void *) t->data, t->size);
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, t->ti);
|
||||
}
|
||||
|
||||
htp_tensor_make_clean(t, ctx->dirty_map);
|
||||
}
|
||||
|
||||
void htp_tensor_flush(struct htp_context * ctx, const struct htp_tensor * t) {
|
||||
if (!bitmap_test(ctx->dirty_map, t->ti)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (t->size > HEX_L2_FLUSH_ALL_THRESHOLD) {
|
||||
flush_all_dcache(ctx);
|
||||
return;
|
||||
}
|
||||
|
||||
flush_tensor_range(ctx, t);
|
||||
}
|
||||
|
||||
// One dirty tensor's line-aligned range, placed in the flattened global block space.
|
||||
struct l2flush_range {
|
||||
uint32_t start; // line-aligned start address
|
||||
uint32_t end; // line-aligned end address
|
||||
uint32_t block_first; // global block index of this range's first block
|
||||
uint32_t n_blocks; // number of HEX_L2_BLOCK_SIZE chunks (last may be partial)
|
||||
};
|
||||
|
||||
struct l2flush_multi_task {
|
||||
struct htp_thread_trace * trace;
|
||||
struct l2flush_range ranges[HTP_OP_MAX_INPUTS];
|
||||
uint32_t n_ranges;
|
||||
uint32_t total_blocks;
|
||||
uint32_t blocks_per_thread;
|
||||
};
|
||||
|
||||
static void l2flush_multi_worker(unsigned int n, unsigned int i, void * data) {
|
||||
(void) n;
|
||||
struct l2flush_multi_task * task = (struct l2flush_multi_task *) data;
|
||||
|
||||
const uint32_t gb_first = i * task->blocks_per_thread;
|
||||
uint32_t gb_last = gb_first + task->blocks_per_thread;
|
||||
if (gb_last > task->total_blocks) {
|
||||
gb_last = task->total_blocks;
|
||||
}
|
||||
if (gb_first >= gb_last) {
|
||||
return;
|
||||
}
|
||||
|
||||
struct htp_thread_trace * tr = &task->trace[i];
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, gb_first);
|
||||
|
||||
for (uint32_t r = 0; r < task->n_ranges; r++) {
|
||||
const struct l2flush_range * rg = &task->ranges[r];
|
||||
const uint32_t rb_first = rg->block_first;
|
||||
const uint32_t rb_last = rg->block_first + rg->n_blocks;
|
||||
|
||||
const uint32_t lo = gb_first > rb_first ? gb_first : rb_first;
|
||||
const uint32_t hi = gb_last < rb_last ? gb_last : rb_last;
|
||||
if (lo >= hi) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const uint32_t s = rg->start + (lo - rb_first) * HEX_L2_BLOCK_SIZE;
|
||||
uint32_t e = rg->start + (hi - rb_first) * HEX_L2_BLOCK_SIZE;
|
||||
if (e > rg->end) {
|
||||
e = rg->end;
|
||||
}
|
||||
hex_l2flush((void *) (uintptr_t) s, e - s);
|
||||
}
|
||||
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, gb_first);
|
||||
}
|
||||
|
||||
void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) {
|
||||
uint64_t total_dirty = 0;
|
||||
for (uint32_t i = 0; i < n; i++) {
|
||||
const struct htp_tensor * t = tensors[i];
|
||||
if (t && bitmap_test(ctx->dirty_map, t->ti)) {
|
||||
total_dirty += t->size;
|
||||
}
|
||||
}
|
||||
|
||||
if (total_dirty == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (total_dirty > HEX_L2_FLUSH_ALL_THRESHOLD) {
|
||||
flush_all_dcache(ctx);
|
||||
return;
|
||||
}
|
||||
|
||||
// Aggregate is small enough to walk. Thread it across all dirty ranges at once
|
||||
// when it is worth the dispatch, otherwise flush sequentially.
|
||||
if (total_dirty > HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1) {
|
||||
struct l2flush_multi_task task;
|
||||
task.trace = ctx->trace;
|
||||
task.n_ranges = 0;
|
||||
|
||||
uint32_t block_acc = 0;
|
||||
for (uint32_t i = 0; i < n; i++) {
|
||||
const struct htp_tensor * t = tensors[i];
|
||||
if (!t || !bitmap_test(ctx->dirty_map, t->ti)) {
|
||||
continue;
|
||||
}
|
||||
// Clear as we go: dedups a tensor passed as multiple srcs (e.g. mul(x,x)).
|
||||
htp_tensor_make_clean(t, ctx->dirty_map);
|
||||
|
||||
struct l2flush_range * rg = &task.ranges[task.n_ranges++];
|
||||
rg->start = hex_align_down((size_t) t->data, HEX_L2_LINE_SIZE);
|
||||
rg->end = hex_align_up((size_t) t->data + t->size, HEX_L2_LINE_SIZE);
|
||||
rg->block_first = block_acc;
|
||||
rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE;
|
||||
block_acc += rg->n_blocks;
|
||||
}
|
||||
|
||||
task.total_blocks = block_acc;
|
||||
task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div);
|
||||
|
||||
work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads);
|
||||
return;
|
||||
}
|
||||
|
||||
struct htp_thread_trace * tr = &ctx->trace[0];
|
||||
for (uint32_t i = 0; i < n; i++) {
|
||||
const struct htp_tensor * t = tensors[i];
|
||||
if (!t || !bitmap_test(ctx->dirty_map, t->ti)) {
|
||||
continue;
|
||||
}
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, t->ti);
|
||||
hex_l2flush((void *) t->data, t->size);
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, t->ti);
|
||||
htp_tensor_make_clean(t, ctx->dirty_map);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user