hexagon: add LIGHTNING_INDEXER support (#31289)

* hexagon: initial support for LIGHTNING_INDEXER op

* hex-lght-idxr: hmx for prefill

* hex-lght-idxr: q8_0 decode gather transpose, two-block decode compute

* hex-lght-idxr: pipelined hmx path for decode and prefill

* hex-lid: add support for K (f32)

* hex-lid: add support for K (f32)

* hex-lid: add support for q4_0

* hex-lid: remove scalar VTCM reads in the HMX q4_0 K prep and remove vector spills

* hex-lid: host/kernel kparams alighnment and remove sw devices in kernel

* hex-lit: update dma pipeline to use canonical dbl buffering in all paths

* hex-lit: add GGML_HEXAGON_LI_SELECT to force hmx/hvx paths

* hex-lit: add support for dims collapse

* hex-lit: make kparams constistent with other ops

---------

Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
This commit is contained in:
Todor BoinovskiandMax Krasnyansky authored and GitHub committed 2026-11-03 21:17:30 -08:00
1 parent 149f5641b9
commit bee23322f1
13 files changed
+2217

No files matched your search

+3
View File
@@ -135,6 +135,7 @@ size_t opt_mbuf_total = 0; // total buffer space limit (
int opt_mm_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported)
int opt_fa_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported)
int opt_fa_split = 1; // 1 = head-split + team-split when n_kv_heads < N_cores (default), 0 = off
int opt_li_select = 2; // 2 = HMX -> HVX, 1 = HVX, 0 = CPU (unsupported)
int opt_gdn_select = 2; // 2 = HMX -> HVX, 1 = HVX, 0 = CPU (unsupported)
int opt_ar_select = 2; // 2 = fused ALLREDUCE+ADD (default), 1 = unfused ALLREDUCE, 0 = fallback to CPY+FENCE
int opt_ar_scatter = 1; // 1 = reduce-scatter the fused ALLREDUCE+ADD (default), 0 = full reduction
@@ -3409,6 +3410,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {
const char * str_mm_select = getenv("GGML_HEXAGON_MM_SELECT");
const char * str_fa_select = getenv("GGML_HEXAGON_FA_SELECT");
const char * str_fa_split = getenv("GGML_HEXAGON_FA_SPLIT");
const char * str_li_select = getenv("GGML_HEXAGON_LI_SELECT");
const char * str_gdn_select = getenv("GGML_HEXAGON_GDN_SELECT");
const char * str_ar_select = getenv("GGML_HEXAGON_AR_SELECT");
const char * str_ar_scatter = getenv("GGML_HEXAGON_AR_SCATTER");
@@ -3475,6 +3477,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {
opt_mm_select = str_mm_select ? atoi(str_mm_select) : opt_mm_select;
opt_fa_select = str_fa_select ? atoi(str_fa_select) : opt_fa_select;
opt_fa_split = str_fa_split ? atoi(str_fa_split) : opt_fa_split;
opt_li_select = str_li_select ? atoi(str_li_select) : opt_li_select;
opt_gdn_select = str_gdn_select ? atoi(str_gdn_select) : opt_gdn_select;
opt_ar_select = str_ar_select ? atoi(str_ar_select) : opt_ar_select;
opt_ar_scatter = str_ar_scatter ? atoi(str_ar_scatter) : opt_ar_scatter;
+1
View File
@@ -29,6 +29,7 @@ extern size_t opt_mbuf_total;
extern int opt_mm_select;
extern int opt_fa_select;
extern int opt_fa_split;
extern int opt_li_select;
extern int opt_gdn_select;
extern int opt_ar_select;
extern int opt_ar_scatter;
+258
View File
@@ -64,6 +64,7 @@ htp_op_code htp_op_remap(const struct ggml_tensor * t) {
case GGML_OP_CONV_2D_DW: return HTP_OP_CONV_2D_DW;
case GGML_OP_ROLL: return HTP_OP_ROLL;
case GGML_OP_UPSCALE: return HTP_OP_UPSCALE;
case GGML_OP_LIGHTNING_INDEXER: return HTP_OP_LIGHTNING_INDEXER;
case GGML_OP_UNARY:
switch (ggml_get_unary_op(t)) {
@@ -4708,6 +4709,233 @@ static inline bool htp_kparams_solve_dsv4_hc_post(
return htp_kparams_solve_dsv4_hc_post_impl(sess, op->src[0], op->src[1], op->src[2], op->src[3], op, kparams);
}
static bool htp_lightning_indexer_can_collapse(
const struct ggml_tensor * q,
const struct ggml_tensor * k,
const struct ggml_tensor * w,
const struct ggml_tensor * m,
const struct ggml_tensor * dst
) {
if (!q || !k || !w || !m || !dst) {
return false;
}
// 2D K normalization: ne[1] is n_kv, ne[2] is 1
const bool can_collapse_k = (k->ne[1] > 1 && k->ne[2] == 1);
// Stream-to-token collapse across q, w, dst (and m)
bool can_collapse_stream = false;
if (q->ne[3] > 1 && k->ne[3] == 1) {
const bool q_ok = (q->nb[3] == (size_t) q->ne[2] * q->nb[2]);
const bool dst_ok = (dst->ne[3] == q->ne[3] && dst->ne[2] == 1 && dst->ne[1] == q->ne[2] &&
dst->nb[3] == (size_t) dst->ne[1] * dst->nb[1]);
const bool w_ok = (w->ne[3] == q->ne[3] && w->ne[2] == 1 && w->ne[1] == q->ne[2] &&
w->nb[3] == (size_t) w->ne[1] * w->nb[1]) ||
(w->ne[3] == 1 && w->ne[1] == q->ne[2] * q->ne[3]);
const bool m_ok = (m->ne[3] == q->ne[3] && m->ne[2] == 1 && m->ne[1] == q->ne[2] &&
m->nb[3] == (size_t) m->ne[1] * m->nb[1]) ||
(m->ne[3] == 1 && m->ne[1] == q->ne[2] * q->ne[3]);
can_collapse_stream = q_ok && dst_ok && w_ok && m_ok;
}
return can_collapse_k || can_collapse_stream;
}
static void htp_lightning_indexer_collapse_tensors(
const struct ggml_tensor * q,
const struct ggml_tensor * k,
const struct ggml_tensor * w,
const struct ggml_tensor * m,
const struct ggml_tensor * dst,
struct ggml_tensor * q_c,
struct ggml_tensor * k_c,
struct ggml_tensor * w_c,
struct ggml_tensor * m_c,
struct ggml_tensor * dst_c
) {
*q_c = *q;
*k_c = *k;
*w_c = *w;
*m_c = *m;
*dst_c = *dst;
if (k->ne[1] > 1 && k->ne[2] == 1) {
k_c->ne[2] = k->ne[1];
k_c->ne[1] = 1;
k_c->nb[1] = k->nb[1];
k_c->nb[2] = k->nb[1];
k_c->nb[3] = k->nb[2];
}
if (q->ne[3] > 1 && k->ne[3] == 1) {
const int64_t n_tokens_collapsed = (int64_t) q->ne[2] * q->ne[3];
q_c->ne[2] = n_tokens_collapsed;
q_c->ne[3] = 1;
q_c->nb[3] = (size_t) q_c->ne[2] * q_c->nb[2];
if (w->ne[3] == q->ne[3]) {
w_c->ne[1] = n_tokens_collapsed;
w_c->ne[3] = 1;
w_c->nb[2] = (size_t) w_c->ne[1] * w_c->nb[1];
w_c->nb[3] = w_c->nb[2];
}
dst_c->ne[1] = n_tokens_collapsed;
dst_c->ne[3] = 1;
dst_c->nb[2] = (size_t) dst_c->ne[1] * dst_c->nb[1];
dst_c->nb[3] = dst_c->nb[2];
if (m->ne[3] == q->ne[3] && m->ne[1] == q->ne[2]) {
m_c->ne[1] = n_tokens_collapsed;
m_c->ne[3] = 1;
m_c->nb[2] = (size_t) m_c->ne[1] * m_c->nb[1];
m_c->nb[3] = m_c->nb[2];
}
}
}
static bool htp_kparams_solve_lightning_indexer_impl(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * q,
const struct ggml_tensor * k,
const struct ggml_tensor * w,
const struct ggml_tensor * m,
const struct ggml_tensor * op,
struct htp_lid_kernel_params * kparams
) {
if (opt_li_select < 1) {
return false;
}
if (k->type != GGML_TYPE_F32 && k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_Q8_0 && k->type != GGML_TYPE_Q4_0) {
return false;
}
if (q->ne[0] % 32 != 0 ||
!ggml_is_contiguous_rows(q) || !ggml_is_contiguous_rows(k) ||
!ggml_is_contiguous_rows(w) || !ggml_is_contiguous_rows(m) ||
!ggml_is_contiguous_rows(op)) {
return false;
}
const uint32_t n_kv_blocks = (k->ne[2] + HTP_LID_KV_BLOCK - 1) / HTP_LID_KV_BLOCK;
if (n_kv_blocks == 0 || q->ne[2] == 0) {
return false;
}
const uint32_t D = q->ne[0];
const uint32_t n_head = q->ne[1];
const uint32_t n_tokens = q->ne[2];
const size_t k_row_size = ggml_row_size(k->type, D);
kparams->k_row_size = (uint32_t) k_row_size;
kparams->div_n_head = init_fastdiv_values(n_head);
kparams->div_m_ne3 = init_fastdiv_values((uint32_t) m->ne[3]);
if (opt_li_select >= 2 && htp_session_n_hmx(sess) > 0 && D % HMX_FP16_TILE_N_COLS == 0) {
const uint32_t n_threads = htp_session_n_threads(sess);
const uint32_t n_stream = q->ne[3];
const uint32_t n_dot = D / HMX_FP16_TILE_N_COLS;
auto vtcm_usage = [&](uint32_t n_tok, uint32_t kv_block, uint32_t n_qsets) {
return htp_lid_hmx_compute_vtcm_usage(D, n_head, n_tok, kv_block, n_qsets, n_threads, k_row_size);
};
uint32_t n_tok = (std::min)(n_tokens, (uint32_t) HTP_LID_HMX_TOKENS_MAX);
while (n_tok > 32 && vtcm_usage(n_tok, 64, 1) > htp_session_vtcm_size(sess)) {
n_tok /= 2;
}
uint32_t n_qsets = n_tok == n_tokens ? (std::min)(n_stream, (uint32_t) HTP_LID_HMX_MAX_QSETS) : 1;
while (n_qsets > 1 && vtcm_usage(n_tok, 64, n_qsets) > htp_session_vtcm_size(sess)) {
n_qsets /= 2;
}
uint32_t n_qbuf = 1;
if (n_qsets == 1 && n_head >= 16 && n_stream * ((n_tokens + n_tok - 1) / n_tok) > 1) {
uint32_t nt2 = n_tok;
while (nt2 > 32 && vtcm_usage(nt2, 64, 2) > htp_session_vtcm_size(sess)) {
nt2 /= 2;
}
const size_t q_bytes = (size_t) nt2 * n_head * D * sizeof(float);
const size_t k_bytes = (size_t) k->ne[2] * ggml_row_size(k->type, D);
if (vtcm_usage(nt2, 64, 2) <= htp_session_vtcm_size(sess) && q_bytes >= k_bytes) {
n_tok = nt2;
n_qbuf = 2;
}
}
const uint32_t n_ch = (n_tok * n_head + 31) / 32;
const uint32_t kv_seg = hex_round_up(k->ne[2], 64);
const uint32_t n_rows = n_qsets * kv_seg;
const uint32_t bc_max = n_rows <= 512 ? n_rows : (std::max)(256u, n_rows / 4);
uint32_t kv_block = 64;
while (kv_block < HTP_LID_HMX_KV_BLOCK_MAX && kv_block < bc_max && (kv_block / 32) * n_ch * (n_dot + 1) < 256 &&
vtcm_usage(n_tok, kv_block * 2, n_qsets * n_qbuf) <= htp_session_vtcm_size(sess)) {
kv_block *= 2;
}
const size_t vtcm_size = vtcm_usage(n_tok, kv_block, n_qsets * n_qbuf);
if (vtcm_size <= htp_session_vtcm_size(sess)) {
const uint32_t n_tt = (n_tokens + n_tok - 1) / n_tok;
kparams->kernel_type = HTP_LID_KERNEL_HMX;
kparams->n_threads = n_threads;
kparams->n_tokens_tile = n_tok;
kparams->n_kv_blocks = (n_rows + kv_block - 1) / kv_block;
kparams->blocks_per_thread = 0;
kparams->kv_block = kv_block;
kparams->n_qsets = n_qsets;
kparams->n_qbuf = n_qbuf;
kparams->kv_seg = kv_seg;
kparams->n_tt = n_tt;
kparams->vtcm_size = (uint32_t) vtcm_size;
kparams->div_kv_seg = init_fastdiv_values(kv_seg);
kparams->div_n_tt = init_fastdiv_values(n_tt);
kparams->div_n_ch = init_fastdiv_values(n_ch);
return true;
}
}
const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), n_kv_blocks);
uint32_t n_tok = (std::min)((uint32_t) q->ne[2], (uint32_t) HTP_LID_TOKENS_MAX);
while (n_tok > 1 && htp_lid_compute_vtcm_usage(D, n_head, n_tok, n_threads, k_row_size) > htp_session_vtcm_size(sess)) {
n_tok /= 2;
}
const size_t vtcm_size = htp_lid_compute_vtcm_usage(D, n_head, n_tok, n_threads, k_row_size);
if (vtcm_size > htp_session_vtcm_size(sess)) {
return false;
}
kparams->kernel_type = HTP_LID_KERNEL_HVX;
kparams->n_threads = n_threads;
kparams->n_tokens_tile = n_tok;
kparams->n_kv_blocks = n_kv_blocks;
kparams->blocks_per_thread = (n_kv_blocks + n_threads - 1) / n_threads;
kparams->kv_block = HTP_LID_KV_BLOCK;
kparams->n_qsets = 1;
kparams->n_qbuf = 1;
kparams->kv_seg = 0;
kparams->n_tt = 0;
kparams->vtcm_size = (uint32_t) vtcm_size;
return true;
}
static bool htp_kparams_solve_lightning_indexer(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * op,
struct htp_lid_kernel_params * kparams
) {
if (htp_lightning_indexer_can_collapse(op->src[0], op->src[1], op->src[2], op->src[3], op)) {
struct ggml_tensor q_c, k_c, w_c, m_c, d_c;
htp_lightning_indexer_collapse_tensors(op->src[0], op->src[1], op->src[2], op->src[3], op,
&q_c, &k_c, &w_c, &m_c, &d_c);
return htp_kparams_solve_lightning_indexer_impl(sess, &q_c, &k_c, &w_c, &m_c, &d_c, kparams);
}
return htp_kparams_solve_lightning_indexer_impl(sess, op->src[0], op->src[1], op->src[2], op->src[3], op, kparams);
}
template <auto SolveFn>
static inline bool htp_kparams_invoke(
const struct ggml_hexagon_session * sess,
@@ -4861,6 +5089,28 @@ void htp_opnode_collapse(const struct ggml_hexagon_session * sess, htp_opnode &
}
return;
}
if (node.opcode == HTP_OP_LIGHTNING_INDEXER) {
if (node.inputs.size() < 4 || !node.inputs[0] || !node.inputs[1] || !node.inputs[2] || !node.inputs[3] || node.outputs.empty() || !node.outputs[0]) {
return;
}
const struct ggml_tensor * q = node.inputs[0];
const struct ggml_tensor * k = node.inputs[1];
const struct ggml_tensor * w = node.inputs[2];
const struct ggml_tensor * m = node.inputs[3];
const struct ggml_tensor * dst = node.outputs[0];
if (htp_lightning_indexer_can_collapse(q, k, w, m, dst)) {
struct ggml_tensor q_c, k_c, w_c, m_c, d_c;
htp_lightning_indexer_collapse_tensors(q, k, w, m, dst, &q_c, &k_c, &w_c, &m_c, &d_c);
node.inputs[0] = node.add_dummy(q_c);
node.inputs[1] = node.add_dummy(k_c);
node.inputs[2] = node.add_dummy(w_c);
node.inputs[3] = node.add_dummy(m_c);
node.outputs[0] = node.add_dummy(d_c);
}
return;
}
}
bool htp_kparams_solve(
@@ -4901,6 +5151,11 @@ bool htp_kparams_solve(
const ggml_tensor * src3 = node.inputs.size() > 3 ? node.inputs[3] : nullptr;
return htp_kparams_solve_dsv4_hc_post_impl(sess, src0, src1, src2, src3, dst, node.kparams<htp_dsv4_hc_post_kernel_params>());
}
if (node.opcode == HTP_OP_LIGHTNING_INDEXER) {
const ggml_tensor * src2 = node.inputs.size() > 2 ? node.inputs[2] : nullptr;
const ggml_tensor * src3 = node.inputs.size() > 3 ? node.inputs[3] : nullptr;
return htp_kparams_solve_lightning_indexer_impl(sess, src0, src1, src2, src3, dst, node.kparams<htp_lid_kernel_params>());
}
return htp_kparams_solve(sess, node.node, node.kernel_params);
}
@@ -5005,6 +5260,9 @@ bool htp_kparams_solve(
case HTP_OP_ROLL:
return htp_kparams_invoke<htp_kparams_solve_roll>(sess, op, kparams);
case HTP_OP_LIGHTNING_INDEXER:
return htp_kparams_invoke<htp_kparams_solve_lightning_indexer>(sess, op, kparams);
case HTP_OP_DIAG:
return htp_kparams_invoke<htp_kparams_solve_diag>(sess, op, kparams);
+1
View File
@@ -18,6 +18,7 @@
#include "htp/get-rows-ops.h"
#include "htp/glu-ops.h"
#include "htp/im2col-ops.h"
#include "htp/lightning-indexer-ops.h"
#include "htp/matmul-ops.h"
#include "htp/pad-ops.h"
#include "htp/pool-ops.h"
+7
View File
@@ -11,6 +11,7 @@
#include "htp/gated-delta-net-ops.h"
#include "htp/get-rows-ops.h"
#include "htp/glu-ops.h"
#include "htp/lightning-indexer-ops.h"
#include "htp/matmul-ops.h"
#include "htp/pad-ops.h"
#include "htp/repeat-ops.h"
@@ -197,6 +198,12 @@ static std::string format_kernel_params(const htp_opnode & node) {
const char * ktype_str = kparams->kernel_type == HTP_DSV4_HC_POST_KERNEL_COMB ? "comb" : "identity";
snprintf(str, max_size, "%s nth %u vtcm %u", ktype_str,
(unsigned int) kparams->n_threads, (unsigned int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_LIGHTNING_INDEXER) {
const auto * kparams = node.kparams<htp_lid_kernel_params>();
snprintf(str, max_size, "%s tok %u bc %u nth %u vtcm %u",
kparams->kernel_type == HTP_LID_KERNEL_HMX ? "hmx" : "hvx",
(unsigned int) kparams->n_tokens_tile, (unsigned int) kparams->kv_block,
(unsigned int) kparams->n_threads, (unsigned int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID ||
node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) {
const auto * kparams = node.kparams<htp_binary_kernel_params>();
+1
View File
@@ -51,6 +51,7 @@ add_library(${HTP_LIB} SHARED
upscale-ops.c
allreduce-ops.c
dsv4-hc-ops.c
lightning-indexer-ops.c
)
target_compile_definitions(${HTP_LIB} PRIVATE
+1
View File
@@ -203,5 +203,6 @@ int op_pool_1d(struct htp_ops_context * octx);
int op_allreduce(struct htp_ops_context * octx);
int op_roll(struct htp_ops_context * octx);
int op_upscale(struct htp_ops_context * octx);
int op_lightning_indexer(struct htp_ops_context * octx);
#endif /* HTP_CTX_H */
+1
View File
@@ -120,6 +120,7 @@ enum htp_op_code {
HTP_OP_DSV4_HC_PRE,
HTP_OP_DSV4_HC_POST,
HTP_OP_TOP_K_MASK,
HTP_OP_LIGHTNING_INDEXER,
HTP_OP_INVALID
};
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,213 @@
#ifndef HTP_LIGHTNING_INDEXER_OPS_H
#define HTP_LIGHTNING_INDEXER_OPS_H
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#include "hex-common.h"
#include "hex-fastdiv.h"
#include "htp-vtcm.h"
#ifdef __cplusplus
extern "C" {
#endif
#define HTP_LID_KV_BLOCK 64 // kv rows per block (HVX and HMX Bc)
#define HTP_LID_TOKENS_MAX 32 // max tokens per Q tile (HVX splat path)
#define HTP_LID_HMX_TOKENS_MAX 512 // HMX can keep more tokens; VTCM decides
#define HTP_LID_HMX_KV_BLOCK_MAX 2048
#define HTP_LID_HMX_MAX_QSETS 8 // streams in one HMX pipeline run
#define HTP_LID_N_BUF 4 // K / mask / output buffers per thread (two blocks computed, two in flight)
#ifndef HMX_FP16_TILE_N_COLS
#define HMX_FP16_TILE_N_COLS 32
#endif
#ifndef HMX_FP16_TILE_SIZE
#define HMX_FP16_TILE_SIZE 2048
#endif
enum htp_lid_kernel_type {
HTP_LID_KERNEL_HVX = 0,
HTP_LID_KERNEL_HMX = 1,
};
struct htp_lid_kernel_params {
uint8_t kernel_type; // enum htp_lid_kernel_type
uint32_t n_threads;
uint32_t n_tokens_tile; // tokens per Q tile
uint32_t n_kv_blocks; // HVX: per stream, HMX: per run of n_qsets sets
uint32_t blocks_per_thread;
uint32_t vtcm_size;
uint32_t k_row_size; // K row bytes
uint32_t kv_block; // HMX: kv rows per block (multiple of 64)
uint32_t n_qsets; // HMX: Q sets kept in VTCM (streams run in one pipeline)
uint32_t n_qbuf; // HMX: Q buffers, 2: the next run's Q is prepped during the current run
uint32_t kv_seg; // HMX: rows per set, n_kv padded to 64
uint32_t n_tt; // HMX: token tiles per stream
struct fastdiv_values div_n_head;
struct fastdiv_values div_m_ne3; // mask streams (broadcast)
struct fastdiv_values div_kv_seg; // HMX
struct fastdiv_values div_n_tt; // HMX
struct fastdiv_values div_n_ch; // HMX: channel tiles per Q set
};
#if defined(__cplusplus)
static_assert(sizeof(struct htp_lid_kernel_params) <= 128, "htp_lid_kernel_params is too large");
#else
_Static_assert(sizeof(struct htp_lid_kernel_params) <= 128, "htp_lid_kernel_params is too large");
#endif
// VTCM layout shared by host (fit check) and device (buffer offsets).
// Shared regions: Q splats (one f16 vector per q element) and head weight splats for the current token tile.
// Per-thread regions: Q prep staging, K / mask / output buffers, two transposed K blocks and transpose scratch.
struct htp_lid_vtcm_layout {
size_t off_q;
size_t off_w;
size_t off_qs;
size_t off_k;
size_t off_kt;
size_t off_m;
size_t off_o;
size_t size_q_row; // f16 Q row, padded
size_t size_qs_row; // f32 Q staging row, padded
size_t size_k_row; // f16 K row, padded
size_t size_k_raw_row;
size_t size_m_row; // f16 mask row of one kv block, padded
size_t size_o_row; // f32 output row of one kv block, padded
size_t size_qs; // per thread
size_t size_k_buf; // one K buffer
size_t size_kt; // per thread: two transposed K blocks, two 64x64 scratch tiles, Q8_0 block scales
size_t size_m_buf; // one mask buffer
size_t size_o_buf; // one output buffer
size_t total_bytes;
};
static inline void htp_lid_vtcm_layout_build(struct htp_lid_vtcm_layout * L, size_t D, size_t n_head, size_t n_tok, size_t n_threads, size_t k_row_size) {
L->size_q_row = hex_round_up(D * sizeof(uint16_t), 128);
L->size_qs_row = hex_round_up(D * sizeof(float), 128);
L->size_k_row = hex_round_up(D * sizeof(uint16_t), 128);
L->size_k_raw_row = L->size_k_row > hex_round_up(k_row_size, 128) ? L->size_k_row : hex_round_up(k_row_size, 128);
L->size_m_row = hex_round_up(HTP_LID_KV_BLOCK * sizeof(uint16_t), 128);
L->size_o_row = hex_round_up(HTP_LID_KV_BLOCK * sizeof(float), 128);
L->size_qs = 2 * (n_head * L->size_qs_row + hex_round_up(n_head * sizeof(float), 128)) + L->size_q_row;
L->size_k_buf = HTP_LID_KV_BLOCK * L->size_k_raw_row;
L->size_kt = (2 * hex_round_up(D, HTP_LID_KV_BLOCK) + 2 * HTP_LID_KV_BLOCK + D / 32) * 128;
L->size_m_buf = n_tok * L->size_m_row;
L->size_o_buf = n_tok * L->size_o_row;
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_q, n_tok * n_head * D * 128);
VTCM_LAYOUT_ALLOC(off, off_w, n_tok * n_head * 128);
VTCM_LAYOUT_ALLOC(off, off_qs, n_threads * L->size_qs);
VTCM_LAYOUT_ALLOC(off, off_k, n_threads * HTP_LID_N_BUF * L->size_k_buf);
VTCM_LAYOUT_ALLOC(off, off_kt, n_threads * L->size_kt);
VTCM_LAYOUT_ALLOC(off, off_m, n_threads * HTP_LID_N_BUF * L->size_m_buf);
VTCM_LAYOUT_ALLOC(off, off_o, n_threads * HTP_LID_N_BUF * L->size_o_buf);
L->total_bytes = off;
}
static inline size_t htp_lid_compute_vtcm_usage(size_t D, size_t n_head, size_t n_tok, size_t n_threads, size_t k_row_size) {
struct htp_lid_vtcm_layout L;
htp_lid_vtcm_layout_build(&L, D, n_head, n_tok, n_threads, k_row_size);
return L.total_bytes;
}
// HMX path. K is the activation (kv rows), Q the weights (one output channel per (token, head)).
// mm1: S = relu(K Q^T) * |w| per channel (HMX convert scale + pos clamp)
// mm2: O = S G, G[(t,h), t] = sign(w[t,h]). O is [kv, token], f16.
// Q weights, G, scales and w are per Q set (one per stream when all tokens fit one tile).
struct htp_lid_hmx_vtcm_layout {
size_t off_qw; // Q weight tiles [n_qsets][n_ch][n_dot]
size_t off_g; // G weight tiles [n_qsets][n_ch]
size_t off_sc; // per channel scales, 256 bytes per channel tile, [n_qsets][n_ch]
size_t off_sc_id; // identity scales
size_t off_s; // S tiles of one block and token tile, [n_rt][n_sg]
size_t off_w; // w of the token tile, [n_qsets][n_tok][n_head] f32
size_t off_qs; // per thread: f32 and f16 staging for 32 Q rows, gather scratch
size_t off_k_raw[2]; // K rows, [Bc][size_k_row]
size_t off_k_t[2]; // K tiles [n_rt][n_dot]
size_t off_o[2]; // O tiles [n_rt][n_u]
size_t off_m[4]; // mask rows [n_tok][size_m_row]
size_t off_out[2]; // f32 output rows [n_tok][size_o_row]
size_t size_qs_row; // f32 Q row, padded
size_t size_qh_row; // f16 Q row, padded
size_t size_k_row; // f16 K row, padded
size_t size_k_raw_row;
size_t size_m_row;
size_t size_o_row;
size_t size_qs; // per thread
size_t size_w; // per Q set
uint32_t n_ch; // channel tiles: ceil(n_tok * n_head / 32)
uint32_t n_u; // token tiles: ceil(n_tok / 32)
uint32_t n_dot; // D / 32
uint32_t n_rt; // kv row tiles per block: Bc / 32
uint32_t n_sg; // S tiles per (row tile, token tile): min(n_head, n_ch)
size_t total_bytes;
};
static inline void htp_lid_hmx_vtcm_layout_build(struct htp_lid_hmx_vtcm_layout * L,
size_t D, size_t n_head, size_t n_tok, size_t Bc, size_t n_qsets, size_t n_threads, size_t k_row_size) {
L->n_ch = (uint32_t) ((n_tok * n_head + 31) / 32);
L->n_u = (uint32_t) ((n_tok + 31) / 32);
L->n_dot = (uint32_t) (D / 32);
L->n_rt = (uint32_t) (Bc / 32);
L->n_sg = (uint32_t) (n_head < L->n_ch ? n_head : L->n_ch);
L->size_qs_row = hex_round_up(D * sizeof(float), 128);
L->size_qh_row = hex_round_up(D * sizeof(uint16_t), 128);
L->size_k_row = hex_round_up(D * sizeof(uint16_t), 128);
L->size_k_raw_row = L->size_k_row > hex_round_up(k_row_size, 128) ? L->size_k_row : hex_round_up(k_row_size, 128);
L->size_m_row = hex_round_up(Bc * sizeof(uint16_t), 128);
L->size_o_row = hex_round_up(Bc * sizeof(float), 128);
L->size_qs = 2 * 32 * L->size_qs_row + 32 * L->size_qh_row + hex_round_up(Bc / 64 * 128, 128);
L->size_w = hex_round_up(n_tok * n_head * sizeof(float), 128);
const size_t tile = HMX_FP16_TILE_SIZE;
// tile regions first: they need 2 KB alignment
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_qw, n_qsets * L->n_ch * L->n_dot * tile);
VTCM_LAYOUT_ALLOC(off, off_g, n_qsets * L->n_ch * tile);
VTCM_LAYOUT_ALLOC(off, off_s, (size_t) L->n_rt * L->n_sg * tile);
VTCM_LAYOUT_ALLOC(off, off_k_t[0], (size_t) L->n_rt * L->n_dot * tile);
VTCM_LAYOUT_ALLOC(off, off_k_t[1], (size_t) L->n_rt * L->n_dot * tile);
VTCM_LAYOUT_ALLOC(off, off_o[0], (size_t) L->n_rt * L->n_u * tile);
VTCM_LAYOUT_ALLOC(off, off_o[1], (size_t) L->n_rt * L->n_u * tile);
VTCM_LAYOUT_ALLOC(off, off_sc, n_qsets * L->n_ch * 256);
VTCM_LAYOUT_ALLOC(off, off_sc_id, 256);
VTCM_LAYOUT_ALLOC(off, off_w, n_qsets * L->size_w);
VTCM_LAYOUT_ALLOC(off, off_qs, n_threads * L->size_qs);
for (int i = 0; i < 2; i++) {
VTCM_LAYOUT_ALLOC(off, off_k_raw[i], Bc * L->size_k_raw_row);
}
for (int i = 0; i < 4; i++) {
VTCM_LAYOUT_ALLOC(off, off_m[i], n_tok * L->size_m_row);
}
for (int i = 0; i < 2; i++) {
VTCM_LAYOUT_ALLOC(off, off_out[i], n_tok * L->size_o_row);
}
L->total_bytes = off;
}
static inline size_t htp_lid_hmx_compute_vtcm_usage(size_t D, size_t n_head, size_t n_tok, size_t Bc, size_t n_qsets, size_t n_threads, size_t k_row_size) {
struct htp_lid_hmx_vtcm_layout L;
htp_lid_hmx_vtcm_layout_build(&L, D, n_head, n_tok, Bc, n_qsets, n_threads, k_row_size);
return L.total_bytes;
}
#ifdef __cplusplus
}
#endif
#endif /* HTP_LIGHTNING_INDEXER_OPS_H */
+3
View File
@@ -1056,6 +1056,9 @@ static int execute_op(struct htp_ops_context * octx) {
case HTP_OP_GATED_DELTA_NET:
return op_gated_delta_net(octx);
case HTP_OP_LIGHTNING_INDEXER:
return op_lightning_indexer(octx);
case HTP_OP_TRI:
return op_unary(octx);
+3
View File
@@ -33,6 +33,7 @@ MANAGED_ENV_NAMES = (
"GGML_HEXAGON_MM_SELECT",
"GGML_HEXAGON_FA_SELECT",
"GGML_HEXAGON_FA_HEAD_SPLIT",
"GGML_HEXAGON_LI_SELECT",
"GGML_HEXAGON_GDN_SELECT",
"GGML_HEXAGON_AR_SELECT",
"GGML_HEXAGON_AR_SCATTER",
@@ -235,6 +236,7 @@ def main():
parser.add_argument("--hex-mm-select", help="Select MUL_MAT and MUL_MAT_ID kernel (GGML_HEXAGON_MM_SELECT) 2:HMX,1:HVX,0:disable")
parser.add_argument("--hex-fa-select", help="Select Flash Attention kernel (GGML_HEXAGON_FA_SELECT) 2:HMX,1:HVX,0:disable")
parser.add_argument("--hex-fa-head-split", help="Enable (1) or disable (0) head-parallel flash_attn partitioning (GGML_HEXAGON_FA_HEAD_SPLIT)")
parser.add_argument("--hex-li-select", help="Select Lightning Indexer kernel (GGML_HEXAGON_LI_SELECT) 2:HMX,1:HVX,0:disable")
parser.add_argument("--hex-gdn-select", help="Select Gated Delta Net kernel (GGML_HEXAGON_GDN_SELECT) 2:HMX,1:HVX,0:disable")
parser.add_argument("--hex-ar-select", help="Select All-Reduce kernel (GGML_HEXAGON_AR_SELECT) 1:enable,0:disable")
parser.add_argument("--hex-ar-scatter", help="Enable (1) or disable (0) reduce-scatter for fused ALLREDUCE+ADD (GGML_HEXAGON_AR_SCATTER)")
@@ -388,6 +390,7 @@ def main():
set_env("GGML_HEXAGON_MM_SELECT", args.hex_mm_select)
set_env("GGML_HEXAGON_FA_SELECT", args.hex_fa_select)
set_env("GGML_HEXAGON_FA_HEAD_SPLIT", args.hex_fa_head_split)
set_env("GGML_HEXAGON_LI_SELECT", args.hex_li_select)
set_env("GGML_HEXAGON_GDN_SELECT", args.hex_gdn_select)
set_env("GGML_HEXAGON_AR_SELECT", args.hex_ar_select)
set_env("GGML_HEXAGON_AR_SCATTER", args.hex_ar_scatter)
+3
View File
@@ -12795,6 +12795,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
test_cases.emplace_back(new test_dsv4_hc_post(2560, n_tokens, true)); // synthetic, Qwen4-Exp kernel without the gate chain
}
// Qwen3.8-Flash-Next pooled indexer prefill.
test_cases.emplace_back(new test_lightning_indexer(128, 4, 1088, 1024, 1, 1, GGML_TYPE_F32));
test_cases.emplace_back(new test_lightning_indexer(128, 4, 1088, 1024, 1, 1, GGML_TYPE_Q4_0));
// SWIGLU at a 27B-class FFN width, fused [gate|up] vs split operands
// note: same bytes either way, so a backend that indexes them differently shows it here
for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32, GGML_TYPE_BF16}) {