hexagon: add LIGHTNING_INDEXER support (#31289)
* hexagon: initial support for LIGHTNING_INDEXER op * hex-lght-idxr: hmx for prefill * hex-lght-idxr: q8_0 decode gather transpose, two-block decode compute * hex-lght-idxr: pipelined hmx path for decode and prefill * hex-lid: add support for K (f32) * hex-lid: add support for K (f32) * hex-lid: add support for q4_0 * hex-lid: remove scalar VTCM reads in the HMX q4_0 K prep and remove vector spills * hex-lid: host/kernel kparams alighnment and remove sw devices in kernel * hex-lit: update dma pipeline to use canonical dbl buffering in all paths * hex-lit: add GGML_HEXAGON_LI_SELECT to force hmx/hvx paths * hex-lit: add support for dims collapse * hex-lit: make kparams constistent with other ops --------- Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
This commit is contained in:
1 parent
149f5641b9
commit
bee23322f1
13 files changed
+2217
No files matched your search
@@ -135,6 +135,7 @@ size_t opt_mbuf_total = 0; // total buffer space limit (
|
||||
int opt_mm_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported)
|
||||
int opt_fa_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported)
|
||||
int opt_fa_split = 1; // 1 = head-split + team-split when n_kv_heads < N_cores (default), 0 = off
|
||||
int opt_li_select = 2; // 2 = HMX -> HVX, 1 = HVX, 0 = CPU (unsupported)
|
||||
int opt_gdn_select = 2; // 2 = HMX -> HVX, 1 = HVX, 0 = CPU (unsupported)
|
||||
int opt_ar_select = 2; // 2 = fused ALLREDUCE+ADD (default), 1 = unfused ALLREDUCE, 0 = fallback to CPY+FENCE
|
||||
int opt_ar_scatter = 1; // 1 = reduce-scatter the fused ALLREDUCE+ADD (default), 0 = full reduction
|
||||
@@ -3409,6 +3410,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {
|
||||
const char * str_mm_select = getenv("GGML_HEXAGON_MM_SELECT");
|
||||
const char * str_fa_select = getenv("GGML_HEXAGON_FA_SELECT");
|
||||
const char * str_fa_split = getenv("GGML_HEXAGON_FA_SPLIT");
|
||||
const char * str_li_select = getenv("GGML_HEXAGON_LI_SELECT");
|
||||
const char * str_gdn_select = getenv("GGML_HEXAGON_GDN_SELECT");
|
||||
const char * str_ar_select = getenv("GGML_HEXAGON_AR_SELECT");
|
||||
const char * str_ar_scatter = getenv("GGML_HEXAGON_AR_SCATTER");
|
||||
@@ -3475,6 +3477,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {
|
||||
opt_mm_select = str_mm_select ? atoi(str_mm_select) : opt_mm_select;
|
||||
opt_fa_select = str_fa_select ? atoi(str_fa_select) : opt_fa_select;
|
||||
opt_fa_split = str_fa_split ? atoi(str_fa_split) : opt_fa_split;
|
||||
opt_li_select = str_li_select ? atoi(str_li_select) : opt_li_select;
|
||||
opt_gdn_select = str_gdn_select ? atoi(str_gdn_select) : opt_gdn_select;
|
||||
opt_ar_select = str_ar_select ? atoi(str_ar_select) : opt_ar_select;
|
||||
opt_ar_scatter = str_ar_scatter ? atoi(str_ar_scatter) : opt_ar_scatter;
|
||||
|
||||
@@ -29,6 +29,7 @@ extern size_t opt_mbuf_total;
|
||||
extern int opt_mm_select;
|
||||
extern int opt_fa_select;
|
||||
extern int opt_fa_split;
|
||||
extern int opt_li_select;
|
||||
extern int opt_gdn_select;
|
||||
extern int opt_ar_select;
|
||||
extern int opt_ar_scatter;
|
||||
|
||||
@@ -64,6 +64,7 @@ htp_op_code htp_op_remap(const struct ggml_tensor * t) {
|
||||
case GGML_OP_CONV_2D_DW: return HTP_OP_CONV_2D_DW;
|
||||
case GGML_OP_ROLL: return HTP_OP_ROLL;
|
||||
case GGML_OP_UPSCALE: return HTP_OP_UPSCALE;
|
||||
case GGML_OP_LIGHTNING_INDEXER: return HTP_OP_LIGHTNING_INDEXER;
|
||||
|
||||
case GGML_OP_UNARY:
|
||||
switch (ggml_get_unary_op(t)) {
|
||||
@@ -4708,6 +4709,233 @@ static inline bool htp_kparams_solve_dsv4_hc_post(
|
||||
return htp_kparams_solve_dsv4_hc_post_impl(sess, op->src[0], op->src[1], op->src[2], op->src[3], op, kparams);
|
||||
}
|
||||
|
||||
static bool htp_lightning_indexer_can_collapse(
|
||||
const struct ggml_tensor * q,
|
||||
const struct ggml_tensor * k,
|
||||
const struct ggml_tensor * w,
|
||||
const struct ggml_tensor * m,
|
||||
const struct ggml_tensor * dst
|
||||
) {
|
||||
if (!q || !k || !w || !m || !dst) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 2D K normalization: ne[1] is n_kv, ne[2] is 1
|
||||
const bool can_collapse_k = (k->ne[1] > 1 && k->ne[2] == 1);
|
||||
|
||||
// Stream-to-token collapse across q, w, dst (and m)
|
||||
bool can_collapse_stream = false;
|
||||
if (q->ne[3] > 1 && k->ne[3] == 1) {
|
||||
const bool q_ok = (q->nb[3] == (size_t) q->ne[2] * q->nb[2]);
|
||||
const bool dst_ok = (dst->ne[3] == q->ne[3] && dst->ne[2] == 1 && dst->ne[1] == q->ne[2] &&
|
||||
dst->nb[3] == (size_t) dst->ne[1] * dst->nb[1]);
|
||||
const bool w_ok = (w->ne[3] == q->ne[3] && w->ne[2] == 1 && w->ne[1] == q->ne[2] &&
|
||||
w->nb[3] == (size_t) w->ne[1] * w->nb[1]) ||
|
||||
(w->ne[3] == 1 && w->ne[1] == q->ne[2] * q->ne[3]);
|
||||
const bool m_ok = (m->ne[3] == q->ne[3] && m->ne[2] == 1 && m->ne[1] == q->ne[2] &&
|
||||
m->nb[3] == (size_t) m->ne[1] * m->nb[1]) ||
|
||||
(m->ne[3] == 1 && m->ne[1] == q->ne[2] * q->ne[3]);
|
||||
can_collapse_stream = q_ok && dst_ok && w_ok && m_ok;
|
||||
}
|
||||
|
||||
return can_collapse_k || can_collapse_stream;
|
||||
}
|
||||
|
||||
static void htp_lightning_indexer_collapse_tensors(
|
||||
const struct ggml_tensor * q,
|
||||
const struct ggml_tensor * k,
|
||||
const struct ggml_tensor * w,
|
||||
const struct ggml_tensor * m,
|
||||
const struct ggml_tensor * dst,
|
||||
struct ggml_tensor * q_c,
|
||||
struct ggml_tensor * k_c,
|
||||
struct ggml_tensor * w_c,
|
||||
struct ggml_tensor * m_c,
|
||||
struct ggml_tensor * dst_c
|
||||
) {
|
||||
*q_c = *q;
|
||||
*k_c = *k;
|
||||
*w_c = *w;
|
||||
*m_c = *m;
|
||||
*dst_c = *dst;
|
||||
|
||||
if (k->ne[1] > 1 && k->ne[2] == 1) {
|
||||
k_c->ne[2] = k->ne[1];
|
||||
k_c->ne[1] = 1;
|
||||
k_c->nb[1] = k->nb[1];
|
||||
k_c->nb[2] = k->nb[1];
|
||||
k_c->nb[3] = k->nb[2];
|
||||
}
|
||||
|
||||
if (q->ne[3] > 1 && k->ne[3] == 1) {
|
||||
const int64_t n_tokens_collapsed = (int64_t) q->ne[2] * q->ne[3];
|
||||
|
||||
q_c->ne[2] = n_tokens_collapsed;
|
||||
q_c->ne[3] = 1;
|
||||
q_c->nb[3] = (size_t) q_c->ne[2] * q_c->nb[2];
|
||||
|
||||
if (w->ne[3] == q->ne[3]) {
|
||||
w_c->ne[1] = n_tokens_collapsed;
|
||||
w_c->ne[3] = 1;
|
||||
w_c->nb[2] = (size_t) w_c->ne[1] * w_c->nb[1];
|
||||
w_c->nb[3] = w_c->nb[2];
|
||||
}
|
||||
|
||||
dst_c->ne[1] = n_tokens_collapsed;
|
||||
dst_c->ne[3] = 1;
|
||||
dst_c->nb[2] = (size_t) dst_c->ne[1] * dst_c->nb[1];
|
||||
dst_c->nb[3] = dst_c->nb[2];
|
||||
|
||||
if (m->ne[3] == q->ne[3] && m->ne[1] == q->ne[2]) {
|
||||
m_c->ne[1] = n_tokens_collapsed;
|
||||
m_c->ne[3] = 1;
|
||||
m_c->nb[2] = (size_t) m_c->ne[1] * m_c->nb[1];
|
||||
m_c->nb[3] = m_c->nb[2];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static bool htp_kparams_solve_lightning_indexer_impl(
|
||||
const struct ggml_hexagon_session * sess,
|
||||
const struct ggml_tensor * q,
|
||||
const struct ggml_tensor * k,
|
||||
const struct ggml_tensor * w,
|
||||
const struct ggml_tensor * m,
|
||||
const struct ggml_tensor * op,
|
||||
struct htp_lid_kernel_params * kparams
|
||||
) {
|
||||
if (opt_li_select < 1) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (k->type != GGML_TYPE_F32 && k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_Q8_0 && k->type != GGML_TYPE_Q4_0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (q->ne[0] % 32 != 0 ||
|
||||
!ggml_is_contiguous_rows(q) || !ggml_is_contiguous_rows(k) ||
|
||||
!ggml_is_contiguous_rows(w) || !ggml_is_contiguous_rows(m) ||
|
||||
!ggml_is_contiguous_rows(op)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const uint32_t n_kv_blocks = (k->ne[2] + HTP_LID_KV_BLOCK - 1) / HTP_LID_KV_BLOCK;
|
||||
if (n_kv_blocks == 0 || q->ne[2] == 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const uint32_t D = q->ne[0];
|
||||
const uint32_t n_head = q->ne[1];
|
||||
const uint32_t n_tokens = q->ne[2];
|
||||
const size_t k_row_size = ggml_row_size(k->type, D);
|
||||
|
||||
kparams->k_row_size = (uint32_t) k_row_size;
|
||||
kparams->div_n_head = init_fastdiv_values(n_head);
|
||||
kparams->div_m_ne3 = init_fastdiv_values((uint32_t) m->ne[3]);
|
||||
|
||||
if (opt_li_select >= 2 && htp_session_n_hmx(sess) > 0 && D % HMX_FP16_TILE_N_COLS == 0) {
|
||||
const uint32_t n_threads = htp_session_n_threads(sess);
|
||||
const uint32_t n_stream = q->ne[3];
|
||||
const uint32_t n_dot = D / HMX_FP16_TILE_N_COLS;
|
||||
|
||||
auto vtcm_usage = [&](uint32_t n_tok, uint32_t kv_block, uint32_t n_qsets) {
|
||||
return htp_lid_hmx_compute_vtcm_usage(D, n_head, n_tok, kv_block, n_qsets, n_threads, k_row_size);
|
||||
};
|
||||
|
||||
uint32_t n_tok = (std::min)(n_tokens, (uint32_t) HTP_LID_HMX_TOKENS_MAX);
|
||||
while (n_tok > 32 && vtcm_usage(n_tok, 64, 1) > htp_session_vtcm_size(sess)) {
|
||||
n_tok /= 2;
|
||||
}
|
||||
|
||||
uint32_t n_qsets = n_tok == n_tokens ? (std::min)(n_stream, (uint32_t) HTP_LID_HMX_MAX_QSETS) : 1;
|
||||
while (n_qsets > 1 && vtcm_usage(n_tok, 64, n_qsets) > htp_session_vtcm_size(sess)) {
|
||||
n_qsets /= 2;
|
||||
}
|
||||
|
||||
uint32_t n_qbuf = 1;
|
||||
if (n_qsets == 1 && n_head >= 16 && n_stream * ((n_tokens + n_tok - 1) / n_tok) > 1) {
|
||||
uint32_t nt2 = n_tok;
|
||||
while (nt2 > 32 && vtcm_usage(nt2, 64, 2) > htp_session_vtcm_size(sess)) {
|
||||
nt2 /= 2;
|
||||
}
|
||||
const size_t q_bytes = (size_t) nt2 * n_head * D * sizeof(float);
|
||||
const size_t k_bytes = (size_t) k->ne[2] * ggml_row_size(k->type, D);
|
||||
if (vtcm_usage(nt2, 64, 2) <= htp_session_vtcm_size(sess) && q_bytes >= k_bytes) {
|
||||
n_tok = nt2;
|
||||
n_qbuf = 2;
|
||||
}
|
||||
}
|
||||
|
||||
const uint32_t n_ch = (n_tok * n_head + 31) / 32;
|
||||
const uint32_t kv_seg = hex_round_up(k->ne[2], 64);
|
||||
const uint32_t n_rows = n_qsets * kv_seg;
|
||||
const uint32_t bc_max = n_rows <= 512 ? n_rows : (std::max)(256u, n_rows / 4);
|
||||
uint32_t kv_block = 64;
|
||||
while (kv_block < HTP_LID_HMX_KV_BLOCK_MAX && kv_block < bc_max && (kv_block / 32) * n_ch * (n_dot + 1) < 256 &&
|
||||
vtcm_usage(n_tok, kv_block * 2, n_qsets * n_qbuf) <= htp_session_vtcm_size(sess)) {
|
||||
kv_block *= 2;
|
||||
}
|
||||
|
||||
const size_t vtcm_size = vtcm_usage(n_tok, kv_block, n_qsets * n_qbuf);
|
||||
if (vtcm_size <= htp_session_vtcm_size(sess)) {
|
||||
const uint32_t n_tt = (n_tokens + n_tok - 1) / n_tok;
|
||||
kparams->kernel_type = HTP_LID_KERNEL_HMX;
|
||||
kparams->n_threads = n_threads;
|
||||
kparams->n_tokens_tile = n_tok;
|
||||
kparams->n_kv_blocks = (n_rows + kv_block - 1) / kv_block;
|
||||
kparams->blocks_per_thread = 0;
|
||||
kparams->kv_block = kv_block;
|
||||
kparams->n_qsets = n_qsets;
|
||||
kparams->n_qbuf = n_qbuf;
|
||||
kparams->kv_seg = kv_seg;
|
||||
kparams->n_tt = n_tt;
|
||||
kparams->vtcm_size = (uint32_t) vtcm_size;
|
||||
kparams->div_kv_seg = init_fastdiv_values(kv_seg);
|
||||
kparams->div_n_tt = init_fastdiv_values(n_tt);
|
||||
kparams->div_n_ch = init_fastdiv_values(n_ch);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), n_kv_blocks);
|
||||
uint32_t n_tok = (std::min)((uint32_t) q->ne[2], (uint32_t) HTP_LID_TOKENS_MAX);
|
||||
while (n_tok > 1 && htp_lid_compute_vtcm_usage(D, n_head, n_tok, n_threads, k_row_size) > htp_session_vtcm_size(sess)) {
|
||||
n_tok /= 2;
|
||||
}
|
||||
|
||||
const size_t vtcm_size = htp_lid_compute_vtcm_usage(D, n_head, n_tok, n_threads, k_row_size);
|
||||
if (vtcm_size > htp_session_vtcm_size(sess)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
kparams->kernel_type = HTP_LID_KERNEL_HVX;
|
||||
kparams->n_threads = n_threads;
|
||||
kparams->n_tokens_tile = n_tok;
|
||||
kparams->n_kv_blocks = n_kv_blocks;
|
||||
kparams->blocks_per_thread = (n_kv_blocks + n_threads - 1) / n_threads;
|
||||
kparams->kv_block = HTP_LID_KV_BLOCK;
|
||||
kparams->n_qsets = 1;
|
||||
kparams->n_qbuf = 1;
|
||||
kparams->kv_seg = 0;
|
||||
kparams->n_tt = 0;
|
||||
kparams->vtcm_size = (uint32_t) vtcm_size;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool htp_kparams_solve_lightning_indexer(
|
||||
const struct ggml_hexagon_session * sess,
|
||||
const struct ggml_tensor * op,
|
||||
struct htp_lid_kernel_params * kparams
|
||||
) {
|
||||
if (htp_lightning_indexer_can_collapse(op->src[0], op->src[1], op->src[2], op->src[3], op)) {
|
||||
struct ggml_tensor q_c, k_c, w_c, m_c, d_c;
|
||||
htp_lightning_indexer_collapse_tensors(op->src[0], op->src[1], op->src[2], op->src[3], op,
|
||||
&q_c, &k_c, &w_c, &m_c, &d_c);
|
||||
return htp_kparams_solve_lightning_indexer_impl(sess, &q_c, &k_c, &w_c, &m_c, &d_c, kparams);
|
||||
}
|
||||
return htp_kparams_solve_lightning_indexer_impl(sess, op->src[0], op->src[1], op->src[2], op->src[3], op, kparams);
|
||||
}
|
||||
template <auto SolveFn>
|
||||
static inline bool htp_kparams_invoke(
|
||||
const struct ggml_hexagon_session * sess,
|
||||
@@ -4861,6 +5089,28 @@ void htp_opnode_collapse(const struct ggml_hexagon_session * sess, htp_opnode &
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (node.opcode == HTP_OP_LIGHTNING_INDEXER) {
|
||||
if (node.inputs.size() < 4 || !node.inputs[0] || !node.inputs[1] || !node.inputs[2] || !node.inputs[3] || node.outputs.empty() || !node.outputs[0]) {
|
||||
return;
|
||||
}
|
||||
const struct ggml_tensor * q = node.inputs[0];
|
||||
const struct ggml_tensor * k = node.inputs[1];
|
||||
const struct ggml_tensor * w = node.inputs[2];
|
||||
const struct ggml_tensor * m = node.inputs[3];
|
||||
const struct ggml_tensor * dst = node.outputs[0];
|
||||
|
||||
if (htp_lightning_indexer_can_collapse(q, k, w, m, dst)) {
|
||||
struct ggml_tensor q_c, k_c, w_c, m_c, d_c;
|
||||
htp_lightning_indexer_collapse_tensors(q, k, w, m, dst, &q_c, &k_c, &w_c, &m_c, &d_c);
|
||||
node.inputs[0] = node.add_dummy(q_c);
|
||||
node.inputs[1] = node.add_dummy(k_c);
|
||||
node.inputs[2] = node.add_dummy(w_c);
|
||||
node.inputs[3] = node.add_dummy(m_c);
|
||||
node.outputs[0] = node.add_dummy(d_c);
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
bool htp_kparams_solve(
|
||||
@@ -4901,6 +5151,11 @@ bool htp_kparams_solve(
|
||||
const ggml_tensor * src3 = node.inputs.size() > 3 ? node.inputs[3] : nullptr;
|
||||
return htp_kparams_solve_dsv4_hc_post_impl(sess, src0, src1, src2, src3, dst, node.kparams<htp_dsv4_hc_post_kernel_params>());
|
||||
}
|
||||
if (node.opcode == HTP_OP_LIGHTNING_INDEXER) {
|
||||
const ggml_tensor * src2 = node.inputs.size() > 2 ? node.inputs[2] : nullptr;
|
||||
const ggml_tensor * src3 = node.inputs.size() > 3 ? node.inputs[3] : nullptr;
|
||||
return htp_kparams_solve_lightning_indexer_impl(sess, src0, src1, src2, src3, dst, node.kparams<htp_lid_kernel_params>());
|
||||
}
|
||||
|
||||
return htp_kparams_solve(sess, node.node, node.kernel_params);
|
||||
}
|
||||
@@ -5005,6 +5260,9 @@ bool htp_kparams_solve(
|
||||
case HTP_OP_ROLL:
|
||||
return htp_kparams_invoke<htp_kparams_solve_roll>(sess, op, kparams);
|
||||
|
||||
case HTP_OP_LIGHTNING_INDEXER:
|
||||
return htp_kparams_invoke<htp_kparams_solve_lightning_indexer>(sess, op, kparams);
|
||||
|
||||
case HTP_OP_DIAG:
|
||||
return htp_kparams_invoke<htp_kparams_solve_diag>(sess, op, kparams);
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
#include "htp/get-rows-ops.h"
|
||||
#include "htp/glu-ops.h"
|
||||
#include "htp/im2col-ops.h"
|
||||
#include "htp/lightning-indexer-ops.h"
|
||||
#include "htp/matmul-ops.h"
|
||||
#include "htp/pad-ops.h"
|
||||
#include "htp/pool-ops.h"
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
#include "htp/gated-delta-net-ops.h"
|
||||
#include "htp/get-rows-ops.h"
|
||||
#include "htp/glu-ops.h"
|
||||
#include "htp/lightning-indexer-ops.h"
|
||||
#include "htp/matmul-ops.h"
|
||||
#include "htp/pad-ops.h"
|
||||
#include "htp/repeat-ops.h"
|
||||
@@ -197,6 +198,12 @@ static std::string format_kernel_params(const htp_opnode & node) {
|
||||
const char * ktype_str = kparams->kernel_type == HTP_DSV4_HC_POST_KERNEL_COMB ? "comb" : "identity";
|
||||
snprintf(str, max_size, "%s nth %u vtcm %u", ktype_str,
|
||||
(unsigned int) kparams->n_threads, (unsigned int) kparams->vtcm_size);
|
||||
} else if (node.opcode == HTP_OP_LIGHTNING_INDEXER) {
|
||||
const auto * kparams = node.kparams<htp_lid_kernel_params>();
|
||||
snprintf(str, max_size, "%s tok %u bc %u nth %u vtcm %u",
|
||||
kparams->kernel_type == HTP_LID_KERNEL_HMX ? "hmx" : "hvx",
|
||||
(unsigned int) kparams->n_tokens_tile, (unsigned int) kparams->kv_block,
|
||||
(unsigned int) kparams->n_threads, (unsigned int) kparams->vtcm_size);
|
||||
} else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID ||
|
||||
node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) {
|
||||
const auto * kparams = node.kparams<htp_binary_kernel_params>();
|
||||
|
||||
@@ -51,6 +51,7 @@ add_library(${HTP_LIB} SHARED
|
||||
upscale-ops.c
|
||||
allreduce-ops.c
|
||||
dsv4-hc-ops.c
|
||||
lightning-indexer-ops.c
|
||||
)
|
||||
|
||||
target_compile_definitions(${HTP_LIB} PRIVATE
|
||||
|
||||
@@ -203,5 +203,6 @@ int op_pool_1d(struct htp_ops_context * octx);
|
||||
int op_allreduce(struct htp_ops_context * octx);
|
||||
int op_roll(struct htp_ops_context * octx);
|
||||
int op_upscale(struct htp_ops_context * octx);
|
||||
int op_lightning_indexer(struct htp_ops_context * octx);
|
||||
|
||||
#endif /* HTP_CTX_H */
|
||||
@@ -120,6 +120,7 @@ enum htp_op_code {
|
||||
HTP_OP_DSV4_HC_PRE,
|
||||
HTP_OP_DSV4_HC_POST,
|
||||
HTP_OP_TOP_K_MASK,
|
||||
HTP_OP_LIGHTNING_INDEXER,
|
||||
|
||||
HTP_OP_INVALID
|
||||
};
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,213 @@
|
||||
#ifndef HTP_LIGHTNING_INDEXER_OPS_H
|
||||
#define HTP_LIGHTNING_INDEXER_OPS_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
#include "hex-common.h"
|
||||
#include "hex-fastdiv.h"
|
||||
#include "htp-vtcm.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define HTP_LID_KV_BLOCK 64 // kv rows per block (HVX and HMX Bc)
|
||||
#define HTP_LID_TOKENS_MAX 32 // max tokens per Q tile (HVX splat path)
|
||||
#define HTP_LID_HMX_TOKENS_MAX 512 // HMX can keep more tokens; VTCM decides
|
||||
#define HTP_LID_HMX_KV_BLOCK_MAX 2048
|
||||
#define HTP_LID_HMX_MAX_QSETS 8 // streams in one HMX pipeline run
|
||||
#define HTP_LID_N_BUF 4 // K / mask / output buffers per thread (two blocks computed, two in flight)
|
||||
|
||||
#ifndef HMX_FP16_TILE_N_COLS
|
||||
#define HMX_FP16_TILE_N_COLS 32
|
||||
#endif
|
||||
#ifndef HMX_FP16_TILE_SIZE
|
||||
#define HMX_FP16_TILE_SIZE 2048
|
||||
#endif
|
||||
|
||||
enum htp_lid_kernel_type {
|
||||
HTP_LID_KERNEL_HVX = 0,
|
||||
HTP_LID_KERNEL_HMX = 1,
|
||||
};
|
||||
|
||||
struct htp_lid_kernel_params {
|
||||
uint8_t kernel_type; // enum htp_lid_kernel_type
|
||||
uint32_t n_threads;
|
||||
uint32_t n_tokens_tile; // tokens per Q tile
|
||||
uint32_t n_kv_blocks; // HVX: per stream, HMX: per run of n_qsets sets
|
||||
uint32_t blocks_per_thread;
|
||||
uint32_t vtcm_size;
|
||||
uint32_t k_row_size; // K row bytes
|
||||
uint32_t kv_block; // HMX: kv rows per block (multiple of 64)
|
||||
uint32_t n_qsets; // HMX: Q sets kept in VTCM (streams run in one pipeline)
|
||||
uint32_t n_qbuf; // HMX: Q buffers, 2: the next run's Q is prepped during the current run
|
||||
uint32_t kv_seg; // HMX: rows per set, n_kv padded to 64
|
||||
uint32_t n_tt; // HMX: token tiles per stream
|
||||
|
||||
struct fastdiv_values div_n_head;
|
||||
struct fastdiv_values div_m_ne3; // mask streams (broadcast)
|
||||
struct fastdiv_values div_kv_seg; // HMX
|
||||
struct fastdiv_values div_n_tt; // HMX
|
||||
struct fastdiv_values div_n_ch; // HMX: channel tiles per Q set
|
||||
};
|
||||
|
||||
#if defined(__cplusplus)
|
||||
static_assert(sizeof(struct htp_lid_kernel_params) <= 128, "htp_lid_kernel_params is too large");
|
||||
#else
|
||||
_Static_assert(sizeof(struct htp_lid_kernel_params) <= 128, "htp_lid_kernel_params is too large");
|
||||
#endif
|
||||
|
||||
// VTCM layout shared by host (fit check) and device (buffer offsets).
|
||||
// Shared regions: Q splats (one f16 vector per q element) and head weight splats for the current token tile.
|
||||
// Per-thread regions: Q prep staging, K / mask / output buffers, two transposed K blocks and transpose scratch.
|
||||
struct htp_lid_vtcm_layout {
|
||||
size_t off_q;
|
||||
size_t off_w;
|
||||
size_t off_qs;
|
||||
size_t off_k;
|
||||
size_t off_kt;
|
||||
size_t off_m;
|
||||
size_t off_o;
|
||||
|
||||
size_t size_q_row; // f16 Q row, padded
|
||||
size_t size_qs_row; // f32 Q staging row, padded
|
||||
size_t size_k_row; // f16 K row, padded
|
||||
size_t size_k_raw_row;
|
||||
size_t size_m_row; // f16 mask row of one kv block, padded
|
||||
size_t size_o_row; // f32 output row of one kv block, padded
|
||||
|
||||
size_t size_qs; // per thread
|
||||
size_t size_k_buf; // one K buffer
|
||||
size_t size_kt; // per thread: two transposed K blocks, two 64x64 scratch tiles, Q8_0 block scales
|
||||
size_t size_m_buf; // one mask buffer
|
||||
size_t size_o_buf; // one output buffer
|
||||
|
||||
size_t total_bytes;
|
||||
};
|
||||
|
||||
static inline void htp_lid_vtcm_layout_build(struct htp_lid_vtcm_layout * L, size_t D, size_t n_head, size_t n_tok, size_t n_threads, size_t k_row_size) {
|
||||
L->size_q_row = hex_round_up(D * sizeof(uint16_t), 128);
|
||||
L->size_qs_row = hex_round_up(D * sizeof(float), 128);
|
||||
L->size_k_row = hex_round_up(D * sizeof(uint16_t), 128);
|
||||
L->size_k_raw_row = L->size_k_row > hex_round_up(k_row_size, 128) ? L->size_k_row : hex_round_up(k_row_size, 128);
|
||||
L->size_m_row = hex_round_up(HTP_LID_KV_BLOCK * sizeof(uint16_t), 128);
|
||||
L->size_o_row = hex_round_up(HTP_LID_KV_BLOCK * sizeof(float), 128);
|
||||
|
||||
L->size_qs = 2 * (n_head * L->size_qs_row + hex_round_up(n_head * sizeof(float), 128)) + L->size_q_row;
|
||||
L->size_k_buf = HTP_LID_KV_BLOCK * L->size_k_raw_row;
|
||||
L->size_kt = (2 * hex_round_up(D, HTP_LID_KV_BLOCK) + 2 * HTP_LID_KV_BLOCK + D / 32) * 128;
|
||||
L->size_m_buf = n_tok * L->size_m_row;
|
||||
L->size_o_buf = n_tok * L->size_o_row;
|
||||
|
||||
size_t off = 0;
|
||||
VTCM_LAYOUT_ALLOC(off, off_q, n_tok * n_head * D * 128);
|
||||
VTCM_LAYOUT_ALLOC(off, off_w, n_tok * n_head * 128);
|
||||
VTCM_LAYOUT_ALLOC(off, off_qs, n_threads * L->size_qs);
|
||||
VTCM_LAYOUT_ALLOC(off, off_k, n_threads * HTP_LID_N_BUF * L->size_k_buf);
|
||||
VTCM_LAYOUT_ALLOC(off, off_kt, n_threads * L->size_kt);
|
||||
VTCM_LAYOUT_ALLOC(off, off_m, n_threads * HTP_LID_N_BUF * L->size_m_buf);
|
||||
VTCM_LAYOUT_ALLOC(off, off_o, n_threads * HTP_LID_N_BUF * L->size_o_buf);
|
||||
|
||||
L->total_bytes = off;
|
||||
}
|
||||
|
||||
static inline size_t htp_lid_compute_vtcm_usage(size_t D, size_t n_head, size_t n_tok, size_t n_threads, size_t k_row_size) {
|
||||
struct htp_lid_vtcm_layout L;
|
||||
htp_lid_vtcm_layout_build(&L, D, n_head, n_tok, n_threads, k_row_size);
|
||||
return L.total_bytes;
|
||||
}
|
||||
|
||||
// HMX path. K is the activation (kv rows), Q the weights (one output channel per (token, head)).
|
||||
// mm1: S = relu(K Q^T) * |w| per channel (HMX convert scale + pos clamp)
|
||||
// mm2: O = S G, G[(t,h), t] = sign(w[t,h]). O is [kv, token], f16.
|
||||
// Q weights, G, scales and w are per Q set (one per stream when all tokens fit one tile).
|
||||
struct htp_lid_hmx_vtcm_layout {
|
||||
size_t off_qw; // Q weight tiles [n_qsets][n_ch][n_dot]
|
||||
size_t off_g; // G weight tiles [n_qsets][n_ch]
|
||||
size_t off_sc; // per channel scales, 256 bytes per channel tile, [n_qsets][n_ch]
|
||||
size_t off_sc_id; // identity scales
|
||||
size_t off_s; // S tiles of one block and token tile, [n_rt][n_sg]
|
||||
size_t off_w; // w of the token tile, [n_qsets][n_tok][n_head] f32
|
||||
size_t off_qs; // per thread: f32 and f16 staging for 32 Q rows, gather scratch
|
||||
size_t off_k_raw[2]; // K rows, [Bc][size_k_row]
|
||||
size_t off_k_t[2]; // K tiles [n_rt][n_dot]
|
||||
size_t off_o[2]; // O tiles [n_rt][n_u]
|
||||
size_t off_m[4]; // mask rows [n_tok][size_m_row]
|
||||
size_t off_out[2]; // f32 output rows [n_tok][size_o_row]
|
||||
|
||||
size_t size_qs_row; // f32 Q row, padded
|
||||
size_t size_qh_row; // f16 Q row, padded
|
||||
size_t size_k_row; // f16 K row, padded
|
||||
size_t size_k_raw_row;
|
||||
size_t size_m_row;
|
||||
size_t size_o_row;
|
||||
size_t size_qs; // per thread
|
||||
size_t size_w; // per Q set
|
||||
|
||||
uint32_t n_ch; // channel tiles: ceil(n_tok * n_head / 32)
|
||||
uint32_t n_u; // token tiles: ceil(n_tok / 32)
|
||||
uint32_t n_dot; // D / 32
|
||||
uint32_t n_rt; // kv row tiles per block: Bc / 32
|
||||
uint32_t n_sg; // S tiles per (row tile, token tile): min(n_head, n_ch)
|
||||
|
||||
size_t total_bytes;
|
||||
};
|
||||
|
||||
static inline void htp_lid_hmx_vtcm_layout_build(struct htp_lid_hmx_vtcm_layout * L,
|
||||
size_t D, size_t n_head, size_t n_tok, size_t Bc, size_t n_qsets, size_t n_threads, size_t k_row_size) {
|
||||
L->n_ch = (uint32_t) ((n_tok * n_head + 31) / 32);
|
||||
L->n_u = (uint32_t) ((n_tok + 31) / 32);
|
||||
L->n_dot = (uint32_t) (D / 32);
|
||||
L->n_rt = (uint32_t) (Bc / 32);
|
||||
L->n_sg = (uint32_t) (n_head < L->n_ch ? n_head : L->n_ch);
|
||||
|
||||
L->size_qs_row = hex_round_up(D * sizeof(float), 128);
|
||||
L->size_qh_row = hex_round_up(D * sizeof(uint16_t), 128);
|
||||
L->size_k_row = hex_round_up(D * sizeof(uint16_t), 128);
|
||||
L->size_k_raw_row = L->size_k_row > hex_round_up(k_row_size, 128) ? L->size_k_row : hex_round_up(k_row_size, 128);
|
||||
L->size_m_row = hex_round_up(Bc * sizeof(uint16_t), 128);
|
||||
L->size_o_row = hex_round_up(Bc * sizeof(float), 128);
|
||||
L->size_qs = 2 * 32 * L->size_qs_row + 32 * L->size_qh_row + hex_round_up(Bc / 64 * 128, 128);
|
||||
L->size_w = hex_round_up(n_tok * n_head * sizeof(float), 128);
|
||||
|
||||
const size_t tile = HMX_FP16_TILE_SIZE;
|
||||
|
||||
// tile regions first: they need 2 KB alignment
|
||||
size_t off = 0;
|
||||
VTCM_LAYOUT_ALLOC(off, off_qw, n_qsets * L->n_ch * L->n_dot * tile);
|
||||
VTCM_LAYOUT_ALLOC(off, off_g, n_qsets * L->n_ch * tile);
|
||||
VTCM_LAYOUT_ALLOC(off, off_s, (size_t) L->n_rt * L->n_sg * tile);
|
||||
VTCM_LAYOUT_ALLOC(off, off_k_t[0], (size_t) L->n_rt * L->n_dot * tile);
|
||||
VTCM_LAYOUT_ALLOC(off, off_k_t[1], (size_t) L->n_rt * L->n_dot * tile);
|
||||
VTCM_LAYOUT_ALLOC(off, off_o[0], (size_t) L->n_rt * L->n_u * tile);
|
||||
VTCM_LAYOUT_ALLOC(off, off_o[1], (size_t) L->n_rt * L->n_u * tile);
|
||||
VTCM_LAYOUT_ALLOC(off, off_sc, n_qsets * L->n_ch * 256);
|
||||
VTCM_LAYOUT_ALLOC(off, off_sc_id, 256);
|
||||
VTCM_LAYOUT_ALLOC(off, off_w, n_qsets * L->size_w);
|
||||
VTCM_LAYOUT_ALLOC(off, off_qs, n_threads * L->size_qs);
|
||||
for (int i = 0; i < 2; i++) {
|
||||
VTCM_LAYOUT_ALLOC(off, off_k_raw[i], Bc * L->size_k_raw_row);
|
||||
}
|
||||
for (int i = 0; i < 4; i++) {
|
||||
VTCM_LAYOUT_ALLOC(off, off_m[i], n_tok * L->size_m_row);
|
||||
}
|
||||
for (int i = 0; i < 2; i++) {
|
||||
VTCM_LAYOUT_ALLOC(off, off_out[i], n_tok * L->size_o_row);
|
||||
}
|
||||
|
||||
L->total_bytes = off;
|
||||
}
|
||||
|
||||
static inline size_t htp_lid_hmx_compute_vtcm_usage(size_t D, size_t n_head, size_t n_tok, size_t Bc, size_t n_qsets, size_t n_threads, size_t k_row_size) {
|
||||
struct htp_lid_hmx_vtcm_layout L;
|
||||
htp_lid_hmx_vtcm_layout_build(&L, D, n_head, n_tok, Bc, n_qsets, n_threads, k_row_size);
|
||||
return L.total_bytes;
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* HTP_LIGHTNING_INDEXER_OPS_H */
|
||||
@@ -1056,6 +1056,9 @@ static int execute_op(struct htp_ops_context * octx) {
|
||||
case HTP_OP_GATED_DELTA_NET:
|
||||
return op_gated_delta_net(octx);
|
||||
|
||||
case HTP_OP_LIGHTNING_INDEXER:
|
||||
return op_lightning_indexer(octx);
|
||||
|
||||
case HTP_OP_TRI:
|
||||
return op_unary(octx);
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@ MANAGED_ENV_NAMES = (
|
||||
"GGML_HEXAGON_MM_SELECT",
|
||||
"GGML_HEXAGON_FA_SELECT",
|
||||
"GGML_HEXAGON_FA_HEAD_SPLIT",
|
||||
"GGML_HEXAGON_LI_SELECT",
|
||||
"GGML_HEXAGON_GDN_SELECT",
|
||||
"GGML_HEXAGON_AR_SELECT",
|
||||
"GGML_HEXAGON_AR_SCATTER",
|
||||
@@ -235,6 +236,7 @@ def main():
|
||||
parser.add_argument("--hex-mm-select", help="Select MUL_MAT and MUL_MAT_ID kernel (GGML_HEXAGON_MM_SELECT) 2:HMX,1:HVX,0:disable")
|
||||
parser.add_argument("--hex-fa-select", help="Select Flash Attention kernel (GGML_HEXAGON_FA_SELECT) 2:HMX,1:HVX,0:disable")
|
||||
parser.add_argument("--hex-fa-head-split", help="Enable (1) or disable (0) head-parallel flash_attn partitioning (GGML_HEXAGON_FA_HEAD_SPLIT)")
|
||||
parser.add_argument("--hex-li-select", help="Select Lightning Indexer kernel (GGML_HEXAGON_LI_SELECT) 2:HMX,1:HVX,0:disable")
|
||||
parser.add_argument("--hex-gdn-select", help="Select Gated Delta Net kernel (GGML_HEXAGON_GDN_SELECT) 2:HMX,1:HVX,0:disable")
|
||||
parser.add_argument("--hex-ar-select", help="Select All-Reduce kernel (GGML_HEXAGON_AR_SELECT) 1:enable,0:disable")
|
||||
parser.add_argument("--hex-ar-scatter", help="Enable (1) or disable (0) reduce-scatter for fused ALLREDUCE+ADD (GGML_HEXAGON_AR_SCATTER)")
|
||||
@@ -388,6 +390,7 @@ def main():
|
||||
set_env("GGML_HEXAGON_MM_SELECT", args.hex_mm_select)
|
||||
set_env("GGML_HEXAGON_FA_SELECT", args.hex_fa_select)
|
||||
set_env("GGML_HEXAGON_FA_HEAD_SPLIT", args.hex_fa_head_split)
|
||||
set_env("GGML_HEXAGON_LI_SELECT", args.hex_li_select)
|
||||
set_env("GGML_HEXAGON_GDN_SELECT", args.hex_gdn_select)
|
||||
set_env("GGML_HEXAGON_AR_SELECT", args.hex_ar_select)
|
||||
set_env("GGML_HEXAGON_AR_SCATTER", args.hex_ar_scatter)
|
||||
|
||||
@@ -12795,6 +12795,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
|
||||
test_cases.emplace_back(new test_dsv4_hc_post(2560, n_tokens, true)); // synthetic, Qwen4-Exp kernel without the gate chain
|
||||
}
|
||||
|
||||
// Qwen3.8-Flash-Next pooled indexer prefill.
|
||||
test_cases.emplace_back(new test_lightning_indexer(128, 4, 1088, 1024, 1, 1, GGML_TYPE_F32));
|
||||
test_cases.emplace_back(new test_lightning_indexer(128, 4, 1088, 1024, 1, 1, GGML_TYPE_Q4_0));
|
||||
// SWIGLU at a 27B-class FFN width, fused [gate|up] vs split operands
|
||||
// note: same bytes either way, so a backend that indexes them differently shows it here
|
||||
for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32, GGML_TYPE_BF16}) {
|
||||
|
||||
Reference in new issue
Block a user