build_conv_state_at makes a contiguous copy of the conv input tail before writing it into the recurrent state, so every GDN layer issues two device copies per generated token where one is enough. On Qwen3.8-Flash-Next (36 GDN layers) that is 72 copies per token, now 36. Speculative decoding adds one more per rollback slot. ggml_cpy reads the strided view directly, as the shared build_conv_state already does, so the extra copy is dropped.
1561 lines
73 KiB
C++
1561 lines
73 KiB
C++
#include "models.h"
|
|
#include "llama-impl.h"
|
|
#include "llama-memory-hybrid-idx.h"
|
|
#include "llama-memory-recurrent.h"
|
|
|
|
#include <algorithm>
|
|
#include <cinttypes>
|
|
|
|
// bad metadata must be catchable: GGML_ASSERT aborts the whole process
|
|
static void qwen4exp_require_nonzero(const llama_model_loader & ml, llm_kv kid, uint32_t value) {
|
|
if (value == 0) {
|
|
throw std::runtime_error(format("%s must be greater than zero, got %u", ml.llm_kv(kid).c_str(), value));
|
|
}
|
|
}
|
|
|
|
// get_arr() copies a short array as-is, leaving a zero tail the n-gram hash silently drops
|
|
static void qwen4exp_require_arr_len(llama_model_loader & ml, llm_kv kid, uint32_t n_min) {
|
|
uint32_t n_arr = 0;
|
|
ml.get_arr_n(kid, n_arr, true);
|
|
if (n_arr < n_min) {
|
|
throw std::runtime_error(format("%s has %u entries, but at least %u are required",
|
|
ml.llm_kv(kid).c_str(), n_arr, n_min));
|
|
}
|
|
}
|
|
|
|
void llama_model_qwen4exp::load_arch_hparams(llama_model_loader & ml) {
|
|
ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false);
|
|
ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);
|
|
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
|
|
|
|
ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, true);
|
|
|
|
ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);
|
|
ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner);
|
|
ml.get_key(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state);
|
|
ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
|
|
ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);
|
|
qwen4exp_require_nonzero(ml, LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);
|
|
qwen4exp_require_nonzero(ml, LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner);
|
|
qwen4exp_require_nonzero(ml, LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state);
|
|
qwen4exp_require_nonzero(ml, LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
|
|
qwen4exp_require_nonzero(ml, LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);
|
|
|
|
// HC; low_rank is qwen4exp-specific, DeepSeek-V4 leaves it absent (full rank)
|
|
ml.get_key(LLM_KV_HYPER_CONNECTION_COUNT, hparams.dsv4_hc_mult);
|
|
ml.get_key(LLM_KV_HYPER_CONNECTION_LOW_RANK, hparams.hc_low_rank);
|
|
// a count of 1 has nothing to mix: transformers configuration_qwen4_exp.py:196, vLLM
|
|
// config.py:49 and SGLang configs/qwen4_exp.py:38 all raise on hc_count <= 1
|
|
if (hparams.dsv4_hc_mult <= 1) {
|
|
throw std::runtime_error(format("%s must be greater than one, got %u",
|
|
ml.llm_kv(LLM_KV_HYPER_CONNECTION_COUNT).c_str(), hparams.dsv4_hc_mult));
|
|
}
|
|
qwen4exp_require_nonzero(ml, LLM_KV_HYPER_CONNECTION_LOW_RANK, hparams.hc_low_rank);
|
|
hparams.n_embd_out_impl = hparams.dsv4_hc_mult * hparams.n_embd;
|
|
|
|
ml.get_key(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head);
|
|
ml.get_key(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size);
|
|
ml.get_key(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k);
|
|
qwen4exp_require_nonzero(ml, LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head);
|
|
qwen4exp_require_nonzero(ml, LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size);
|
|
qwen4exp_require_nonzero(ml, LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k);
|
|
ml.get_key_or_arr(LLM_KV_ATTENTION_COMPRESS_RATIOS, hparams.dsv4_compress_ratios, hparams.n_layer_all, false);
|
|
|
|
// QSA pools the indexer keys of blocks of compress_ratio cells, one block size for the whole model
|
|
hparams.indexer_kpool = 0;
|
|
for (uint32_t il = 0; il < hparams.n_layer_all; ++il) {
|
|
const uint32_t r = hparams.dsv4_compress_ratios[il];
|
|
if (r == 0) {
|
|
continue;
|
|
}
|
|
if (hparams.indexer_kpool != 0 && r != hparams.indexer_kpool) {
|
|
throw std::runtime_error(format("QSA layers must share one compress ratio, got %u and %u", hparams.indexer_kpool, r));
|
|
}
|
|
hparams.indexer_kpool = r;
|
|
}
|
|
if (hparams.indexer_kpool == 1 || (hparams.indexer_kpool > 0 && hparams.indexer_top_k % hparams.indexer_kpool != 0)) {
|
|
throw std::runtime_error(format("QSA needs a compress ratio above 1 that divides the budget, got %u and %u",
|
|
hparams.indexer_kpool, hparams.indexer_top_k));
|
|
}
|
|
// the reference groups the visible tokens in cache order and always keeps the tail
|
|
hparams.indexer_kpool_row = 2; // raw key | pooled key
|
|
hparams.indexer_kpool_by_order = true;
|
|
hparams.indexer_kpool_select_tail = true;
|
|
|
|
// PLE n-gram hash embeddings; if the key group is absent every field stays zero
|
|
hparams.is_ple_impl.reset();
|
|
hparams.ple_n_heads = 0;
|
|
|
|
uint32_t n_ple = 0;
|
|
ml.get_arr_n(LLM_KV_PLE_LAYERS, n_ple, false);
|
|
if (n_ple > 0) {
|
|
std::vector<uint32_t> ple_layers;
|
|
ml.get_arr(LLM_KV_PLE_LAYERS, ple_layers);
|
|
if (n_ple != 1) {
|
|
// hparams holds one set of hash constants, so several PLE modules cannot be represented
|
|
throw std::runtime_error(format("%s lists %u layers, but only one PLE layer is supported",
|
|
ml.llm_kv(LLM_KV_PLE_LAYERS).c_str(), n_ple));
|
|
}
|
|
for (uint32_t il : ple_layers) {
|
|
if (il >= hparams.n_layer_all) {
|
|
throw std::runtime_error(format("PLE layer %u is out of range", il));
|
|
}
|
|
hparams.is_ple_impl.set(il);
|
|
}
|
|
|
|
ml.get_key(LLM_KV_PLE_NGRAM_SIZE, hparams.ple_ngram_size);
|
|
ml.get_key(LLM_KV_PLE_HEADS_PER_NGRAM, hparams.ple_heads_per_ngram);
|
|
ml.get_key(LLM_KV_PLE_CONV_KERNEL, hparams.ple_conv_kernel);
|
|
ml.get_key(LLM_KV_PLE_EOS_TOKEN_ID, hparams.ple_eos_token_id);
|
|
// optional: files written before this key fall back to the EOS token
|
|
ml.get_key(LLM_KV_PLE_IMAGE_TOKEN_ID, hparams.ple_image_token_id, false);
|
|
ml.get_key(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, hparams.n_embd_per_layer);
|
|
qwen4exp_require_nonzero(ml, LLM_KV_PLE_CONV_KERNEL, hparams.ple_conv_kernel);
|
|
qwen4exp_require_nonzero(ml, LLM_KV_EMBEDDING_LENGTH_PER_LAYER, hparams.n_embd_per_layer);
|
|
|
|
hparams.ple_n_heads = (hparams.ple_ngram_size - 1) * hparams.ple_heads_per_ngram;
|
|
hparams.ple_head_dim = hparams.n_embd_per_layer;
|
|
hparams.ple_embd_token = hparams.ple_image_token_id != 0
|
|
? (llama_token) hparams.ple_image_token_id
|
|
: (llama_token) hparams.ple_eos_token_id;
|
|
if (hparams.ple_ngram_size < 2 || hparams.ple_ngram_size > LLAMA_MAX_PLE_NGRAM) {
|
|
throw std::runtime_error(format("PLE n-gram size %u is out of range", hparams.ple_ngram_size));
|
|
}
|
|
if (hparams.ple_n_heads == 0 || hparams.ple_n_heads > LLAMA_MAX_PLE_HEADS) {
|
|
throw std::runtime_error(format("PLE head count %u is out of range", hparams.ple_n_heads));
|
|
}
|
|
|
|
qwen4exp_require_arr_len(ml, LLM_KV_PLE_LAYER_MULTIPLIERS, hparams.ple_ngram_size);
|
|
qwen4exp_require_arr_len(ml, LLM_KV_PLE_HEAD_OFFSETS, hparams.ple_n_heads);
|
|
qwen4exp_require_arr_len(ml, LLM_KV_PLE_HEAD_VOCAB_SIZES, hparams.ple_n_heads);
|
|
|
|
ml.get_arr(LLM_KV_PLE_LAYER_MULTIPLIERS, hparams.ple_layer_multipliers);
|
|
|
|
// the file stores the head ranges as uint64, so read at that width and narrow to the int32 the gather uses
|
|
std::array<uint64_t, LLAMA_MAX_PLE_HEADS> head_offsets = {};
|
|
std::array<uint64_t, LLAMA_MAX_PLE_HEADS> head_vocab_sizes = {};
|
|
ml.get_arr(LLM_KV_PLE_HEAD_OFFSETS, head_offsets);
|
|
ml.get_arr(LLM_KV_PLE_HEAD_VOCAB_SIZES, head_vocab_sizes);
|
|
for (uint32_t h = 0; h < hparams.ple_n_heads; ++h) {
|
|
if (head_vocab_sizes[h] == 0 ||
|
|
head_offsets[h] > INT32_MAX ||
|
|
head_vocab_sizes[h] > INT32_MAX ||
|
|
head_offsets[h] + head_vocab_sizes[h] > INT32_MAX) {
|
|
throw std::runtime_error(format("PLE head %u range does not fit the int32 row index", h));
|
|
}
|
|
hparams.ple_head_offsets[h] = (uint32_t) head_offsets[h];
|
|
hparams.ple_head_vocab_sizes[h] = (uint32_t) head_vocab_sizes[h];
|
|
}
|
|
}
|
|
|
|
// linear attention everywhere except every full_attention_interval-th layer
|
|
if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer_all, false)) {
|
|
uint32_t full_attn_interval = 4;
|
|
ml.get_key(LLM_KV_FULL_ATTENTION_INTERVAL, full_attn_interval, false);
|
|
qwen4exp_require_nonzero(ml, LLM_KV_FULL_ATTENTION_INTERVAL, full_attn_interval);
|
|
for (uint32_t i = 0; i < hparams.n_layer_all; ++i) {
|
|
hparams.is_recr_impl[i] = (i < hparams.n_layer()) && ((i + 1) % full_attn_interval != 0);
|
|
}
|
|
}
|
|
|
|
// the PLE conv history is a row of the recurrent cache, which linear layers alone have
|
|
for (uint32_t i = 0; i < hparams.n_layer_all; ++i) {
|
|
if (hparams.is_ple(i) && !hparams.is_recr(i)) {
|
|
throw std::runtime_error(format("PLE layer %u is not a linear attention layer", i));
|
|
}
|
|
}
|
|
|
|
switch (hparams.n_layer()) {
|
|
case 48: type = LLM_TYPE_A3B; break;
|
|
default: type = LLM_TYPE_UNKNOWN;
|
|
}
|
|
}
|
|
|
|
void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) {
|
|
LLAMA_LOAD_LOCALS;
|
|
|
|
const int64_t hc = hparams.dsv4_hc_mult;
|
|
const int64_t hc_dim = hc * n_embd;
|
|
const int64_t hc_lr = hparams.hc_low_rank;
|
|
|
|
const auto nf = nextn_flags(ml, LLM_TENSOR_HC_ATTN_NORM);
|
|
const int trunk_flags = nf.trunk;
|
|
const int mtp_flags = nf.mtp;
|
|
|
|
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0);
|
|
|
|
// there is no output_norm: the final hyper-connection mixer carries it
|
|
// the gammas load as [n_embd, hc] so the grouped norm multiplies them without a graph reshape
|
|
hc_head_norm = create_tensor(tn(LLM_TENSOR_HC_HEAD_NORM, "weight"), { n_embd, hc }, trunk_flags | TENSOR_ALLOW_RESHAPE);
|
|
hc_head_down = create_tensor(tn(LLM_TENSOR_HC_HEAD_DOWN, "weight"), { hc_dim, hc_lr }, trunk_flags);
|
|
hc_head_up = create_tensor(tn(LLM_TENSOR_HC_HEAD_UP, "weight"), { hc_lr, hc_dim }, trunk_flags);
|
|
|
|
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED);
|
|
if (output == NULL) {
|
|
output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, TENSOR_DUPLICATED);
|
|
}
|
|
|
|
// flat [ple_head_dim, n_rows] gather target
|
|
if (hparams.ple_n_heads > 0) {
|
|
// the head ranges are what the gather indexes, so they set the minimum row count
|
|
int64_t ple_rows = 0;
|
|
for (uint32_t h = 0; h < hparams.ple_n_heads; ++h) {
|
|
ple_rows = std::max(ple_rows, (int64_t) hparams.ple_head_offsets[h] + hparams.ple_head_vocab_sizes[h]);
|
|
}
|
|
|
|
// the converter pads the table; a model synthesised from metadata has no tensor to ask
|
|
const std::string ple_name = tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight").str();
|
|
if (const auto * ple_w = ml.get_weight(ple_name.c_str())) {
|
|
if (ple_w->tensor->ne[1] < ple_rows) {
|
|
throw std::runtime_error(format("%s has %" PRId64 " rows, too few for the PLE head ranges (%" PRId64 ")",
|
|
ple_name.c_str(), ple_w->tensor->ne[1], ple_rows));
|
|
}
|
|
ple_rows = ple_w->tensor->ne[1];
|
|
}
|
|
|
|
per_layer_tok_embd = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight"),
|
|
{ hparams.ple_head_dim, ple_rows }, TENSOR_READ_LAZY);
|
|
}
|
|
|
|
auto load_block = [&](int il, int flags) {
|
|
auto & layer = layers[il];
|
|
|
|
const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used;
|
|
const int64_t n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff;
|
|
|
|
const int64_t head_k_dim = hparams.ssm_d_state;
|
|
const int64_t head_v_dim = hparams.ssm_d_state;
|
|
const int64_t n_k_heads = hparams.ssm_n_group;
|
|
const int64_t n_v_heads = hparams.ssm_dt_rank;
|
|
const int64_t key_dim = head_k_dim * n_k_heads;
|
|
const int64_t value_dim = head_v_dim * n_v_heads;
|
|
const int64_t conv_dim = key_dim * 2 + value_dim;
|
|
|
|
// matmuls that read the same input: try the merged [a | b] weight the converter writes, else the separate ones
|
|
auto create_merged = [&](llm_tensor t_merged, llm_tensor t_a, llm_tensor t_b, int64_t n_in, int64_t n_a, int64_t n_b,
|
|
ggml_tensor *& merged, ggml_tensor *& a, ggml_tensor *& b) {
|
|
if (flags & TENSOR_SKIP) {
|
|
const int skip = TENSOR_NOT_REQUIRED | TENSOR_SKIP;
|
|
create_tensor(tn(t_merged, "weight", il), { n_in, n_a + n_b }, skip | TENSOR_SKIP_IF_VIRTUAL);
|
|
create_tensor(tn(t_a, "weight", il), { n_in, n_a }, skip);
|
|
create_tensor(tn(t_b, "weight", il), { n_in, n_b }, skip);
|
|
return;
|
|
}
|
|
merged = create_tensor(tn(t_merged, "weight", il), { n_in, n_a + n_b }, flags | TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL);
|
|
if (!merged) {
|
|
a = create_tensor(tn(t_a, "weight", il), { n_in, n_a }, flags);
|
|
b = create_tensor(tn(t_b, "weight", il), { n_in, n_b }, flags);
|
|
}
|
|
};
|
|
|
|
// two HC modules per layer: before the token mixer, before the MoE
|
|
layer.hc_attn_norm = create_tensor(tn(LLM_TENSOR_HC_ATTN_NORM, "weight", il), { n_embd, hc }, flags | TENSOR_ALLOW_RESHAPE);
|
|
layer.hc_attn_up = create_tensor(tn(LLM_TENSOR_HC_ATTN_UP, "weight", il), { hc_lr, hc_dim }, flags);
|
|
create_merged(LLM_TENSOR_HC_ATTN_DOWN_INJECT, LLM_TENSOR_HC_ATTN_DOWN, LLM_TENSOR_HC_ATTN_INJECT, hc_dim, hc_lr, hc,
|
|
layer.hc_attn_down_inject, layer.hc_attn_down, layer.hc_attn_inject);
|
|
layer.hc_ffn_norm = create_tensor(tn(LLM_TENSOR_HC_FFN_NORM, "weight", il), { n_embd, hc }, flags | TENSOR_ALLOW_RESHAPE);
|
|
layer.hc_ffn_up = create_tensor(tn(LLM_TENSOR_HC_FFN_UP, "weight", il), { hc_lr, hc_dim }, flags);
|
|
create_merged(LLM_TENSOR_HC_FFN_DOWN_INJECT, LLM_TENSOR_HC_FFN_DOWN, LLM_TENSOR_HC_FFN_INJECT, hc_dim, hc_lr, hc,
|
|
layer.hc_ffn_down_inject, layer.hc_ffn_down, layer.hc_ffn_inject);
|
|
|
|
if (!hparams.is_recr(il)) {
|
|
// full attention: wq holds [q|gate] interleaved per head
|
|
create_tensor_qkv(layer, il, n_embd, n_embd_head_k * n_head * 2, n_embd_k_gqa, n_embd_v_gqa, flags);
|
|
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", il), { n_embd_head_k * n_head, n_embd }, flags);
|
|
|
|
layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", il), { n_embd_head_k }, flags);
|
|
layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", il), { n_embd_head_k }, flags);
|
|
|
|
const int64_t idx_dim = hparams.indexer_head_size;
|
|
layer.index_q_proj = create_tensor(tn(LLM_TENSOR_INDEXER_Q_PROJ, "weight", il), { n_embd, hparams.indexer_n_head * idx_dim }, flags);
|
|
layer.index_k_proj = create_tensor(tn(LLM_TENSOR_INDEXER_K_PROJ, "weight", il), { n_embd, idx_dim }, flags);
|
|
layer.index_q_norm = create_tensor(tn(LLM_TENSOR_INDEXER_Q_NORM, "weight", il), { idx_dim }, flags);
|
|
layer.index_k_norm = create_tensor(tn(LLM_TENSOR_INDEXER_K_NORM, "weight", il), { idx_dim }, flags);
|
|
} else {
|
|
create_merged(LLM_TENSOR_ATTN_QKV_GATE, LLM_TENSOR_ATTN_QKV, LLM_TENSOR_ATTN_GATE, n_embd, key_dim * 2 + value_dim, value_dim,
|
|
layer.wqkv_z, layer.wqkv, layer.wqkv_gate);
|
|
layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "weight", il), { hparams.ssm_d_conv, conv_dim }, flags);
|
|
layer.ssm_dt = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", il), { hparams.ssm_dt_rank }, flags);
|
|
layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, il), { hparams.ssm_dt_rank }, flags);
|
|
create_merged(LLM_TENSOR_SSM_BETA_ALPHA_CAT, LLM_TENSOR_SSM_BETA, LLM_TENSOR_SSM_ALPHA, n_embd, n_v_heads, n_v_heads,
|
|
layer.ssm_beta_alpha_cat, layer.ssm_beta, layer.ssm_alpha);
|
|
layer.ssm_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", il), { head_v_dim }, flags);
|
|
layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, "weight", il), { value_dim, n_embd }, flags);
|
|
}
|
|
|
|
if (hparams.is_ple(il)) {
|
|
layer.ple_key = create_tensor(tn(LLM_TENSOR_PLE_KEY, "weight", il), { n_embd, hc_dim }, flags);
|
|
layer.ple_value = create_tensor(tn(LLM_TENSOR_PLE_VALUE, "weight", il), { n_embd, n_embd }, flags);
|
|
layer.ple_norm_key = create_tensor(tn(LLM_TENSOR_PLE_NORM_KEY, "weight", il), { n_embd, hc }, flags | TENSOR_ALLOW_RESHAPE);
|
|
layer.ple_norm_query = create_tensor(tn(LLM_TENSOR_PLE_NORM_QUERY, "weight", il), { n_embd, hc }, flags | TENSOR_ALLOW_RESHAPE);
|
|
layer.ple_norm_conv = create_tensor(tn(LLM_TENSOR_PLE_NORM_CONV, "weight", il), { n_embd, hc }, flags | TENSOR_ALLOW_RESHAPE);
|
|
layer.ple_conv1d = create_tensor(tn(LLM_TENSOR_PLE_CONV1D, "weight", il), { hparams.ple_conv_kernel, hc_dim }, flags);
|
|
}
|
|
|
|
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", il), { n_embd, n_expert }, flags);
|
|
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), { n_ff_exp, n_embd, n_expert }, flags);
|
|
create_tensor_gate_up_exps(layer, il, n_embd, n_ff_exp, n_expert, flags);
|
|
|
|
layer.ffn_gate_inp_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP_SHEXP, "weight", il), { n_embd }, flags);
|
|
layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", il), { n_embd, n_ff_shexp }, flags);
|
|
layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", il), { n_embd, n_ff_shexp }, flags);
|
|
layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", il), { n_ff_shexp, n_embd }, flags);
|
|
};
|
|
|
|
for (int il = 0; il < n_layer; ++il) {
|
|
load_block(il, trunk_flags);
|
|
}
|
|
|
|
// the MTP block: one full-attention QSA layer fed by [enorm(e) ; hnorm(h)_s] -> eh_proj per hc stream
|
|
for (int il = n_layer; il < n_layer_all; ++il) {
|
|
load_block(il, mtp_flags);
|
|
|
|
auto & nextn = layers[il].nextn;
|
|
nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", il), { 2*n_embd, n_embd }, mtp_flags);
|
|
nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", il), { n_embd }, mtp_flags);
|
|
// RMS per hc stream of the trunk residual, so the gammas load as [n_embd, hc] like the mixer norms
|
|
nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", il), { n_embd, hc }, mtp_flags | TENSOR_ALLOW_RESHAPE);
|
|
nextn.hc_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_HC_HEAD_NORM, "weight", il), { n_embd, hc }, mtp_flags | TENSOR_ALLOW_RESHAPE);
|
|
nextn.hc_head_down = create_tensor(tn(LLM_TENSOR_NEXTN_HC_HEAD_DOWN, "weight", il), { hc_dim, hc_lr }, mtp_flags);
|
|
nextn.hc_head_up = create_tensor(tn(LLM_TENSOR_NEXTN_HC_HEAD_UP, "weight", il), { hc_lr, hc_dim }, mtp_flags);
|
|
}
|
|
}
|
|
|
|
std::unique_ptr<llm_graph_context> llama_model_qwen4exp::build_arch_graph(const llm_graph_params & params) const {
|
|
if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) {
|
|
return std::make_unique<graph_mtp>(*this, params);
|
|
}
|
|
return std::make_unique<graph>(*this, params);
|
|
}
|
|
|
|
// DFlash drafters read the layer inputs as [n_embd, T] rows, so collapse the streams
|
|
static ggml_tensor * hc_stream_mean(ggml_context * ctx, ggml_tensor * x) {
|
|
const int64_t n_embd = x->ne[0];
|
|
const int64_t hc = x->ne[1];
|
|
const int64_t nt = x->ne[2];
|
|
|
|
ggml_tensor * acc = ggml_view_2d(ctx, x, n_embd, nt, x->nb[2], 0);
|
|
for (int64_t s = 1; s < hc; ++s) {
|
|
acc = ggml_add(ctx, acc, ggml_view_2d(ctx, x, n_embd, nt, x->nb[2], s*x->nb[1]));
|
|
}
|
|
|
|
return ggml_scale(ctx, acc, 1.0f / (float) hc);
|
|
}
|
|
|
|
// Hyper-connections keep hc parallel residual streams [n_embd, hc, T] in place of layer norms.
|
|
// build_hc_mix returns the mixed [n_embd, T] stream and the [hc, T] scatter weights.
|
|
llama_model_qwen4exp::graph::hc_mix
|
|
llama_model_qwen4exp::graph::build_hc_mix(
|
|
ggml_tensor * x,
|
|
ggml_tensor * w_norm,
|
|
ggml_tensor * w_down,
|
|
ggml_tensor * w_up,
|
|
ggml_tensor * w_inject,
|
|
ggml_tensor * w_down_inject,
|
|
int il) {
|
|
const int64_t hc = hparams.dsv4_hc_mult;
|
|
const int64_t hc_dim = hc * n_embd;
|
|
const int64_t nt = x->ne[2];
|
|
|
|
// grouped RMSNorm: reduce over one stream, then scale all streams with the [n_embd, hc] gamma
|
|
// the converter folded each gamma to (1 + w)
|
|
ggml_tensor * xn = ggml_mul(ctx0, ggml_rms_norm(ctx0, x, hparams.f_norm_rms_eps), w_norm);
|
|
xn = ggml_reshape_2d(ctx0, xn, hc_dim, nt);
|
|
cb(xn, "hc_norm", il);
|
|
|
|
ggml_tensor * lo = nullptr;
|
|
ggml_tensor * inject = nullptr;
|
|
if (w_down_inject) {
|
|
// one matmul for both: the hc_down rows, then the hc_inject rows
|
|
const int64_t n_lo = w_down_inject->ne[1] - hc;
|
|
ggml_tensor * lo_inject = build_lora_mm(w_down_inject, xn);
|
|
// scale the combined rows before the split; both lo and inject use 1/hc
|
|
lo_inject = ggml_scale(ctx0, lo_inject, 1.0f / (float) hc);
|
|
lo = ggml_view_2d(ctx0, lo_inject, n_lo, nt, lo_inject->nb[1], 0);
|
|
inject = ggml_view_2d(ctx0, lo_inject, hc, nt, lo_inject->nb[1], ggml_row_size(lo_inject->type, n_lo));
|
|
cb(inject, "hc_inject", il);
|
|
} else {
|
|
lo = ggml_scale(ctx0, build_lora_mm(w_down, xn), 1.0f / (float) hc);
|
|
}
|
|
lo = ggml_silu(ctx0, lo);
|
|
ggml_tensor * gate = build_lora_mm(w_up, lo);
|
|
cb(gate, "hc_gate", il);
|
|
|
|
ggml_tensor * mixed = nullptr;
|
|
if (cparams.fused_dsv4_hc_pre && il >= 0) {
|
|
// sigmoid gate and mean over the streams in one op
|
|
mixed = ggml_dsv4_hc_pre_gated(ctx0,
|
|
ggml_reshape_3d(ctx0, xn, n_embd, hc, nt),
|
|
ggml_reshape_3d(ctx0, gate, n_embd, hc, nt), 1.0f / (float) hc);
|
|
res->add_fused_node({LLM_FUSED_OP_DSV4_HC_PRE, mixed, il});
|
|
} else {
|
|
ggml_tensor * gated = ggml_mul(ctx0, xn, ggml_sigmoid(ctx0, gate));
|
|
gated = ggml_reshape_3d(ctx0, gated, n_embd, hc, nt);
|
|
|
|
// collapse the streams by their mean
|
|
mixed = ggml_view_2d(ctx0, gated, n_embd, nt,
|
|
ggml_row_size(gated->type, n_embd) * hc, 0);
|
|
mixed = ggml_cont(ctx0, mixed);
|
|
for (int64_t c = 1; c < hc; ++c) {
|
|
ggml_tensor * s = ggml_view_2d(ctx0, gated, n_embd, nt,
|
|
ggml_row_size(gated->type, n_embd) * hc,
|
|
ggml_row_size(gated->type, n_embd) * c);
|
|
mixed = ggml_add(ctx0, mixed, s);
|
|
}
|
|
mixed = ggml_scale(ctx0, mixed, 1.0f / (float) hc);
|
|
}
|
|
cb(mixed, "hc_mixed", il);
|
|
|
|
if (!w_down_inject && w_inject) {
|
|
inject = ggml_scale(ctx0, build_lora_mm(w_inject, xn), 1.0f / (float) hc);
|
|
cb(inject, "hc_inject", il);
|
|
}
|
|
|
|
return { mixed, inject };
|
|
}
|
|
|
|
ggml_tensor * llama_model_qwen4exp::graph::build_hc_combine(
|
|
ggml_tensor * residual,
|
|
ggml_tensor * block_out,
|
|
ggml_tensor * inject,
|
|
int il) {
|
|
const int64_t hc = hparams.dsv4_hc_mult;
|
|
const int64_t nt = residual->ne[2];
|
|
|
|
// 2*sigmoid centres the scatter weights on 1, so a zero injection is a plain residual add
|
|
ggml_tensor * w = ggml_sigmoid(ctx0, inject);
|
|
w = ggml_scale(ctx0, w, 2.0f);
|
|
|
|
ggml_tensor * cur = nullptr;
|
|
if (cparams.fused_dsv4_hc_post && il >= 0) {
|
|
// identity comb: every stream adds the same block output, scaled by its own weight
|
|
cur = ggml_dsv4_hc_post(ctx0, block_out, residual, w, nullptr);
|
|
res->add_fused_node({LLM_FUSED_OP_DSV4_HC_POST, cur, il});
|
|
} else {
|
|
w = ggml_reshape_3d(ctx0, w, 1, hc, nt);
|
|
|
|
ggml_tensor * b = ggml_reshape_3d(ctx0, block_out, n_embd, 1, nt);
|
|
b = ggml_repeat_4d(ctx0, b, n_embd, hc, nt, 1);
|
|
|
|
cur = ggml_add(ctx0, residual, ggml_mul(ctx0, b, w));
|
|
}
|
|
cb(cur, "hc_combine", il);
|
|
|
|
return cur;
|
|
}
|
|
|
|
llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_params & params) :
|
|
llm_build_delta_net_base(params), model(model) {
|
|
const int64_t hc = hparams.dsv4_hc_mult;
|
|
|
|
GGML_ASSERT(hparams.n_embd_head_v() == hparams.n_embd_head_k());
|
|
|
|
int sections[4];
|
|
std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
|
|
|
|
auto * inp = build_inp_mem_hybrid();
|
|
|
|
// qwen4exp always builds llama_memory_hybrid_idx, so this downcast is safe
|
|
// the indexer cache inside it is absent when the GGUF has no indexer tensors
|
|
const auto * mctx_hyb = static_cast<const llama_memory_hybrid_idx_context *>(inp->mctx);
|
|
|
|
const llama_kv_cache_context * mctx_idx = mctx_hyb->get_idx();
|
|
if (mctx_idx) {
|
|
GGML_ASSERT(mctx_idx->get_n_kv() == inp->mctx->get_attn()->get_n_kv() &&
|
|
"the indexer cache must track the attention cache cell for cell");
|
|
}
|
|
|
|
ggml_tensor * ple_emb = nullptr;
|
|
if (hparams.ple_n_heads > 0) {
|
|
ple_emb = build_inp_ple(mctx_hyb);
|
|
// make sure ple_emb and build_inp_embd are in the same graph split
|
|
ggml_build_forward_expand(gf, ple_emb);
|
|
}
|
|
|
|
ggml_tensor * inpL = build_inp_embd(model.tok_embd);
|
|
cb(inpL, "model.input_embed", -1);
|
|
ggml_build_forward_expand(gf, inpL);
|
|
|
|
// the QSA layers share one set of k-pool inputs
|
|
// the CUDA lightning indexer takes 32 or 64 heads, QSA has a few, so it scores with plain ops
|
|
llm_graph_input_kpool * inp_kpool = nullptr;
|
|
if (mctx_idx && hparams.indexer_kpool > 0) {
|
|
inp_kpool = build_inp_kpool(mctx_hyb);
|
|
}
|
|
|
|
ggml_tensor * inp_pos = build_inp_pos();
|
|
ggml_tensor * inp_out_ids = build_inp_out_ids();
|
|
|
|
// the wide residual starts as hc identical copies of the embedding
|
|
ggml_tensor * res_hc = ggml_repeat_4d(ctx0,
|
|
ggml_reshape_3d(ctx0, inpL, n_embd, 1, n_tokens),
|
|
n_embd, hc, n_tokens, 1);
|
|
cb(res_hc, "hc_init", -1);
|
|
// make sure hc_init is in the same graph split as the first layer (-sm tensor)
|
|
ggml_build_forward_expand(gf, res_hc);
|
|
|
|
for (int il = 0; il < n_layer; ++il) {
|
|
if ((size_t) il < cparams.embeddings_layer_inp.size() && cparams.embeddings_layer_inp[il]) {
|
|
res->t_layer_inp[il] = hc_stream_mean(ctx0, res_hc);
|
|
cb(res->t_layer_inp[il], "layer_inp", il);
|
|
ggml_build_forward_expand(gf, res->t_layer_inp[il]);
|
|
}
|
|
|
|
if (hparams.is_ple(il)) {
|
|
res_hc = build_ple(inp->get_recr(), ple_emb, res_hc, il);
|
|
}
|
|
|
|
hc_mix attn_mix = build_hc_mix(res_hc,
|
|
model.layers[il].hc_attn_norm,
|
|
model.layers[il].hc_attn_down,
|
|
model.layers[il].hc_attn_up,
|
|
model.layers[il].hc_attn_inject,
|
|
model.layers[il].hc_attn_down_inject,
|
|
il);
|
|
ggml_tensor * cur = attn_mix.cur;
|
|
|
|
ggml_build_forward_expand(gf, cur);
|
|
|
|
if (hparams.is_recr(il)) {
|
|
cur = build_layer_attn_linear(inp->get_recr(), cur, il);
|
|
} else {
|
|
cur = build_layer_attn(inp->get_attn(), mctx_hyb, inp_kpool, cur, inp_pos, sections, il);
|
|
}
|
|
|
|
if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {
|
|
// everything below is per token, so drop the rows that produce no output
|
|
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
|
|
attn_mix.inject = ggml_get_rows(ctx0, attn_mix.inject, inp_out_ids);
|
|
|
|
res_hc = ggml_reshape_2d(ctx0, res_hc, n_embd*hc, res_hc->ne[2]);
|
|
res_hc = ggml_get_rows(ctx0, res_hc, inp_out_ids);
|
|
res_hc = ggml_reshape_3d(ctx0, res_hc, n_embd, hc, res_hc->ne[1]);
|
|
}
|
|
|
|
res_hc = build_hc_combine(res_hc, cur, attn_mix.inject, il);
|
|
|
|
hc_mix ffn_mix = build_hc_mix(res_hc,
|
|
model.layers[il].hc_ffn_norm,
|
|
model.layers[il].hc_ffn_down,
|
|
model.layers[il].hc_ffn_up,
|
|
model.layers[il].hc_ffn_inject,
|
|
model.layers[il].hc_ffn_down_inject,
|
|
il);
|
|
cur = ffn_mix.cur;
|
|
|
|
cur = build_layer_ffn(cur, il);
|
|
cb(cur, "ffn_out", il);
|
|
|
|
res_hc = build_hc_combine(res_hc, cur, ffn_mix.inject, il);
|
|
|
|
// "l_last" is the layer output name that build_cvec and imatrix look for
|
|
cb(res_hc, "l_last", il);
|
|
}
|
|
|
|
// the MTP head reads the hc-wide residual, before the final mixer
|
|
res->t_h_nextn = ggml_reshape_2d(ctx0, res_hc, n_embd*hc, res_hc->ne[2]);
|
|
cb(res->t_h_nextn, "h_nextn", -1);
|
|
ggml_build_forward_expand(gf, res->t_h_nextn);
|
|
|
|
if (crop_after_nextn(inp_out_ids)) {
|
|
res_hc = ggml_reshape_2d(ctx0, res_hc, n_embd*hc, res_hc->ne[2]);
|
|
res_hc = ggml_get_rows(ctx0, res_hc, inp_out_ids);
|
|
res_hc = ggml_reshape_3d(ctx0, res_hc, n_embd, hc, res_hc->ne[1]);
|
|
}
|
|
|
|
// the final mixer is the output norm: there is no separate one
|
|
hc_mix head_mix = build_hc_mix(res_hc,
|
|
model.hc_head_norm, model.hc_head_down, model.hc_head_up,
|
|
nullptr, nullptr, -1);
|
|
ggml_tensor * cur = head_mix.cur;
|
|
|
|
cb(cur, "result_norm", -1);
|
|
res->t_embd = cur;
|
|
|
|
cur = build_lora_mm(model.output, cur, model.output_s);
|
|
cb(cur, "result_output", -1);
|
|
res->t_logits = cur;
|
|
}
|
|
|
|
llama_model_qwen4exp::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) :
|
|
graph(model, params, no_build{}) {
|
|
GGML_ASSERT(hparams.n_layer_nextn == 1 && "qwen4exp MTP has a single block");
|
|
ASSERT_EMBD_OR_TOKEN(ubatch);
|
|
|
|
const int64_t hc = hparams.dsv4_hc_mult;
|
|
GGML_ASSERT(hparams.n_embd_out() == (uint32_t) (n_embd*hc) && "qwen4exp MTP hidden width mismatch");
|
|
|
|
const int il = hparams.n_layer();
|
|
const auto & layer = model.layers[il];
|
|
|
|
GGML_ASSERT(layer.nextn.eh_proj && layer.nextn.enorm && layer.nextn.hnorm && layer.nextn.hc_head_norm &&
|
|
"MTP block missing, load the model with MTP enabled");
|
|
|
|
int sections[4];
|
|
std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
|
|
|
|
auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), hparams.n_embd_out());
|
|
|
|
inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
|
|
ggml_set_input(inp->tokens);
|
|
|
|
inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens);
|
|
ggml_set_input(inp->embd);
|
|
|
|
inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_out(), n_tokens);
|
|
ggml_set_input(inp->h);
|
|
ggml_set_name(inp->h, "mtp_h_input");
|
|
|
|
ggml_tensor * tok_embd;
|
|
if (ubatch.token) {
|
|
tok_embd = ggml_get_rows(ctx0, model.tok_embd, inp->tokens);
|
|
} else {
|
|
tok_embd = inp->embd;
|
|
}
|
|
cb(tok_embd, "mtp_tok_embd", il);
|
|
|
|
ggml_tensor * h = inp->h;
|
|
|
|
res->add_input(std::move(inp));
|
|
|
|
auto * inp_hyb = build_inp_mem_hybrid();
|
|
const auto * mctx_hyb = static_cast<const llama_memory_hybrid_idx_context *>(inp_hyb->mctx);
|
|
|
|
// the draft memory has no recurrent layer, but its input still has to be allocated
|
|
ggml_build_forward_expand(gf, inp_hyb->get_recr()->s_copy);
|
|
|
|
llm_graph_input_kpool * inp_kpool = nullptr;
|
|
if (mctx_hyb->get_idx() && hparams.indexer_kpool > 0) {
|
|
GGML_ASSERT(mctx_hyb->get_idx()->get_n_kv() == mctx_hyb->get_attn()->get_n_kv() &&
|
|
"the indexer cache must track the attention cache cell for cell");
|
|
inp_kpool = build_inp_kpool(mctx_hyb);
|
|
}
|
|
|
|
ggml_tensor * inp_pos = build_inp_pos();
|
|
ggml_tensor * inp_out_ids = build_inp_out_ids();
|
|
|
|
ggml_tensor * h_norm = build_norm(ggml_reshape_3d(ctx0, h, n_embd, hc, n_tokens), layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il);
|
|
cb(h_norm, "mtp_hnorm", il);
|
|
|
|
ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il);
|
|
e_norm = ggml_repeat_4d(ctx0, ggml_reshape_3d(ctx0, e_norm, n_embd, 1, n_tokens), n_embd, hc, n_tokens, 1);
|
|
cb(e_norm, "mtp_enorm", il);
|
|
|
|
ggml_tensor * res_hc = build_lora_mm(layer.nextn.eh_proj, ggml_concat(ctx0, e_norm, h_norm, 0)); // [n_embd, hc, n_tokens]
|
|
cb(res_hc, "mtp_eh_proj", il);
|
|
|
|
hc_mix attn_mix = build_hc_mix(res_hc, layer.hc_attn_norm, layer.hc_attn_down, layer.hc_attn_up, layer.hc_attn_inject, layer.hc_attn_down_inject, il);
|
|
ggml_tensor * cur = attn_mix.cur;
|
|
cur = build_layer_attn(inp_hyb->get_attn(), mctx_hyb, inp_kpool, cur, inp_pos, sections, il);
|
|
res_hc = build_hc_combine(res_hc, cur, attn_mix.inject, il);
|
|
|
|
hc_mix ffn_mix = build_hc_mix(res_hc, layer.hc_ffn_norm, layer.hc_ffn_down, layer.hc_ffn_up, layer.hc_ffn_inject, layer.hc_ffn_down_inject, il);
|
|
cur = ffn_mix.cur;
|
|
cur = build_layer_ffn(cur, il);
|
|
res_hc = build_hc_combine(res_hc, cur, ffn_mix.inject, il);
|
|
|
|
// the next draft step reads this residual as its h
|
|
ggml_tensor * flat = ggml_reshape_2d(ctx0, res_hc, n_embd*hc, n_tokens);
|
|
ggml_tensor * flat_out = inp_out_ids ? ggml_get_rows(ctx0, flat, inp_out_ids) : flat;
|
|
res->t_h_nextn = cparams.embeddings_nextn_masked ? flat_out : flat;
|
|
cb(res->t_h_nextn, "h_nextn", il);
|
|
ggml_build_forward_expand(gf, res->t_h_nextn);
|
|
|
|
hc_mix head_mix = build_hc_mix(ggml_reshape_3d(ctx0, flat_out, n_embd, hc, flat_out->ne[1]),
|
|
layer.nextn.hc_head_norm, layer.nextn.hc_head_down, layer.nextn.hc_head_up,
|
|
nullptr, nullptr, il);
|
|
cur = head_mix.cur;
|
|
cb(cur, "result_norm", -1);
|
|
res->t_embd = cur;
|
|
|
|
cur = build_lora_mm(model.output, cur, model.output_s);
|
|
cb(cur, "result_output", -1);
|
|
res->t_logits = cur;
|
|
}
|
|
|
|
std::pair<ggml_tensor *, ggml_tensor *> llama_model_qwen4exp::graph::build_qkvz(
|
|
ggml_tensor * input,
|
|
int il) {
|
|
const int64_t n_seqs = ubatch.n_seqs;
|
|
const int64_t n_seq_tokens = ubatch.n_seq_tokens;
|
|
|
|
if (model.layers[il].wqkv_z) {
|
|
// qkv and z read the same input: one matmul, then the qkv rows and the z rows
|
|
ggml_tensor * qkvz = build_lora_mm(model.layers[il].wqkv_z, input);
|
|
cb(qkvz, "linear_attn_qkvz", il);
|
|
|
|
const int64_t n_z = hparams.ssm_d_state * hparams.ssm_dt_rank;
|
|
const int64_t n_qkv = qkvz->ne[0] - n_z;
|
|
|
|
ggml_tensor * qkv_mixed = ggml_view_3d(ctx0, qkvz, n_qkv, n_seq_tokens, n_seqs, qkvz->nb[1], qkvz->nb[1] * n_seq_tokens, 0);
|
|
cb(qkv_mixed, "linear_attn_qkv_mixed", il);
|
|
|
|
ggml_tensor * z = ggml_view_2d(ctx0, qkvz, n_z, n_seq_tokens * n_seqs, qkvz->nb[1], ggml_row_size(qkvz->type, n_qkv));
|
|
cb(z, "z", il);
|
|
|
|
return { qkv_mixed, z };
|
|
}
|
|
|
|
ggml_tensor * qkv_mixed = build_lora_mm(model.layers[il].wqkv, input, model.layers[il].wqkv_s);
|
|
qkv_mixed = ggml_reshape_3d(ctx0, qkv_mixed, qkv_mixed->ne[0], n_seq_tokens, n_seqs);
|
|
cb(qkv_mixed, "linear_attn_qkv_mixed", il);
|
|
|
|
ggml_tensor * z = build_lora_mm(model.layers[il].wqkv_gate, input, model.layers[il].wqkv_gate_s);
|
|
cb(z, "z", il);
|
|
|
|
return { qkv_mixed, z };
|
|
}
|
|
|
|
ggml_tensor * llama_model_qwen4exp::graph::build_norm_gated(
|
|
ggml_tensor * input,
|
|
ggml_tensor * weights,
|
|
ggml_tensor * gate,
|
|
int layer) {
|
|
// the one numerical difference from Qwen3.5's GDN: sigmoid output gate, not silu
|
|
ggml_tensor * normalized = build_norm(input, weights, nullptr, LLM_NORM_RMS, layer);
|
|
ggml_tensor * gated = ggml_sigmoid(ctx0, gate);
|
|
|
|
return ggml_mul(ctx0, normalized, gated);
|
|
}
|
|
|
|
// QSA k-pool inputs, shared by the QSA layers: blocks of compress_ratio cells in sequence order, see llama_memory_hybrid_idx
|
|
class llama_model_qwen4exp::llm_graph_input_kpool : public llm_graph_input_i {
|
|
public:
|
|
llm_graph_input_kpool(const llama_memory_hybrid_idx_context * mctx, uint32_t kpool) : mctx(mctx), kpool(kpool) {}
|
|
virtual ~llm_graph_input_kpool() = default;
|
|
|
|
void set_input(const llama_ubatch * ubatch) override {
|
|
mctx->get_idx()->set_input_k_idxs(k_idxs, ubatch);
|
|
mctx->set_input_kpool(pool_cells, pool_idxs, pool_mask, tail_idxs, nullptr, new_pool_idxs, new_pool_rep,
|
|
ubatch, new_pool_pos);
|
|
}
|
|
|
|
bool can_reuse(const llm_graph_params & params) override {
|
|
mctx = static_cast<const llama_memory_hybrid_idx_context *>(params.mctx);
|
|
|
|
const auto * idx = mctx->get_idx();
|
|
if (idx == nullptr) {
|
|
return false;
|
|
}
|
|
|
|
bool res = true;
|
|
|
|
res &= k_idxs->ne[0] == params.ubatch.n_tokens;
|
|
res &= pool_cells->ne[0] == mctx->get_n_kpool();
|
|
res &= pool_mask->ne[1] == params.ubatch.n_tokens;
|
|
res &= tail_idxs->ne[1] == params.ubatch.n_tokens;
|
|
// the scatter mask shape follows n_kv
|
|
res &= n_kv == idx->get_n_kv();
|
|
res &= n_new == mctx->get_n_kpool_new();
|
|
|
|
return res;
|
|
}
|
|
|
|
ggml_tensor * k_idxs = nullptr; // I64 [n_tokens]
|
|
ggml_tensor * pool_cells = nullptr; // I32 [n_pool] cell caching each block's pooled key
|
|
ggml_tensor * pool_idxs = nullptr; // I32 [kpool, n_pool] member cells per block, n_kv sentinel for the padded blocks
|
|
ggml_tensor * pool_mask = nullptr; // F16 [n_pool, n_tokens]
|
|
ggml_tensor * tail_idxs = nullptr; // I32 [kpool - 1, n_tokens]
|
|
ggml_tensor * new_pool_idxs = nullptr; // I32 [kpool, n_new] members of the blocks to re-pool this ubatch
|
|
ggml_tensor * new_pool_flat = nullptr; // I32 [kpool*n_new] view of new_pool_idxs, shared by the QSA layers
|
|
ggml_tensor * new_pool_rep = nullptr; // I64 [n_new] cell to write each new pooled key into
|
|
ggml_tensor * new_pool_pos = nullptr; // I32 [4*n_new] M-RoPE position of each new block's first member
|
|
|
|
const llama_memory_hybrid_idx_context * mctx;
|
|
const uint32_t kpool;
|
|
uint32_t n_new = 0; // padded to a stable bound, never below 1
|
|
uint32_t n_sel = 0;
|
|
uint32_t n_kv = 0;
|
|
};
|
|
|
|
llama_model_qwen4exp::llm_graph_input_kpool * llama_model_qwen4exp::graph::build_inp_kpool(const llama_memory_hybrid_idx_context * mctx_hyb) {
|
|
const auto * mctx_idx = mctx_hyb->get_idx();
|
|
GGML_ASSERT(mctx_idx != nullptr);
|
|
|
|
const uint32_t kpool = hparams.indexer_kpool;
|
|
const uint32_t n_pool = mctx_hyb->get_n_kpool();
|
|
|
|
auto inp = std::make_unique<llm_graph_input_kpool>(mctx_hyb, kpool);
|
|
|
|
inp->k_idxs = mctx_idx->build_input_k_idxs(ctx0, ubatch);
|
|
inp->pool_cells = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pool);
|
|
inp->pool_idxs = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, kpool, n_pool);
|
|
inp->pool_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F16, n_pool, n_tokens);
|
|
inp->tail_idxs = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, kpool - 1, n_tokens);
|
|
ggml_set_input(inp->pool_cells);
|
|
ggml_set_input(inp->pool_idxs);
|
|
ggml_set_input(inp->pool_mask);
|
|
ggml_set_input(inp->tail_idxs);
|
|
|
|
// set_input fills them all, so keep them allocated even when no op reads them
|
|
ggml_build_forward_expand(gf, inp->pool_cells);
|
|
ggml_build_forward_expand(gf, inp->pool_idxs);
|
|
ggml_build_forward_expand(gf, inp->pool_mask);
|
|
ggml_build_forward_expand(gf, inp->tail_idxs);
|
|
|
|
inp->n_kv = mctx_idx->get_n_kv();
|
|
inp->n_new = mctx_hyb->get_n_kpool_new();
|
|
// the top blocks plus the tail
|
|
inp->n_sel = kpool*std::min<uint32_t>(n_pool, hparams.indexer_top_k / kpool) + kpool - 1;
|
|
|
|
inp->new_pool_idxs = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, kpool, inp->n_new);
|
|
ggml_set_input(inp->new_pool_idxs);
|
|
// one view for all layers: each view of a host input is a separate upload
|
|
inp->new_pool_flat = ggml_reshape_1d(ctx0, inp->new_pool_idxs, kpool*inp->n_new);
|
|
// one scatter row per new pool, each a distinct rep row (see kpool_build_state)
|
|
inp->new_pool_rep = ggml_new_tensor_1d(ctx0, GGML_TYPE_I64, inp->n_new);
|
|
ggml_set_input(inp->new_pool_rep);
|
|
inp->new_pool_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 4*inp->n_new);
|
|
ggml_set_input(inp->new_pool_pos);
|
|
|
|
return (llm_graph_input_kpool *) res->add_input(std::move(inp));
|
|
}
|
|
|
|
// QSA attends to the top blocks of compress_ratio cells plus the incomplete tail, like the glm5-next k-pool indexer
|
|
// a block is scored by one pooled key: the mean of its raw indexer keys, normed and rotated to its first member
|
|
ggml_tensor * llama_model_qwen4exp::graph::build_qsa_sel(
|
|
const llama_memory_hybrid_idx_context * mctx_hyb,
|
|
llm_graph_input_kpool * inp_kpool,
|
|
ggml_tensor * cur,
|
|
ggml_tensor * inp_pos,
|
|
int * sections,
|
|
int il) {
|
|
const llama_kv_cache_context * mctx_idx = mctx_hyb->get_idx();
|
|
|
|
const int64_t idx_dim = hparams.indexer_head_size;
|
|
const int64_t n_idx_h = hparams.indexer_n_head;
|
|
const int64_t kpool = inp_kpool->kpool;
|
|
const int64_t n_pool = inp_kpool->pool_cells->ne[0];
|
|
const int64_t n_new = inp_kpool->n_new;
|
|
|
|
GGML_ASSERT(hparams.dsv4_compress_ratios[il] == kpool);
|
|
|
|
// cache rows store raw key | pooled key: pooling precedes norm and rotation, so the raw key gets neither
|
|
ggml_tensor * k_raw = build_lora_mm(model.layers[il].index_k_proj, cur);
|
|
cb(k_raw, "indexer_k_raw", il);
|
|
|
|
ggml_tensor * pzero = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, idx_dim, n_tokens), 0.0f);
|
|
ggml_tensor * packed = ggml_reshape_3d(ctx0, ggml_concat(ctx0, k_raw, pzero, 0), 2*idx_dim, 1, n_tokens);
|
|
ggml_build_forward_expand(gf, mctx_idx->cpy_k(ctx0, packed, inp_kpool->k_idxs, il));
|
|
|
|
// the raw keys and the persistent pooled slots, see llama_memory_hybrid_idx::mem_idx_stale
|
|
auto kpool_cache = mctx_hyb->get_kpool_access(ctx0, il, idx_dim);
|
|
|
|
// pool only the blocks this ubatch completes or regroups
|
|
ggml_tensor * rows = kpool_cache.gather_key_gate(inp_kpool->new_pool_flat);
|
|
rows = ggml_reshape_3d(ctx0, rows, idx_dim, kpool, n_new);
|
|
|
|
// mean over the members; kpool is small, so summing slices beats a transpose plus sum_rows
|
|
ggml_tensor * pooled_new = nullptr;
|
|
for (int64_t i = 0; i < kpool; ++i) {
|
|
ggml_tensor * slice = ggml_view_2d(ctx0, rows, idx_dim, n_new, rows->nb[2], i*rows->nb[1]);
|
|
pooled_new = pooled_new ? ggml_add(ctx0, pooled_new, slice) : ggml_cont(ctx0, slice);
|
|
}
|
|
pooled_new = ggml_scale(ctx0, pooled_new, 1.0f/(float) kpool);
|
|
pooled_new = build_norm(pooled_new, model.layers[il].index_k_norm, nullptr, LLM_NORM_RMS, il);
|
|
|
|
pooled_new = ggml_reshape_3d(ctx0, pooled_new, idx_dim, 1, n_new);
|
|
pooled_new = ggml_rope_multi(ctx0, pooled_new, inp_kpool->new_pool_pos, nullptr,
|
|
n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
ext_factor, attn_factor, beta_fast, beta_slow);
|
|
pooled_new = ggml_reshape_2d(ctx0, pooled_new, idx_dim, n_new);
|
|
cb(pooled_new, "indexer_pool_k_new", il);
|
|
|
|
// scatter the fresh pooled keys into their rep rows, then gather all n_pool of them by cell:
|
|
// the older pools come from the rows earlier ubatches wrote
|
|
ggml_build_forward_expand(gf, kpool_cache.scatter_pooled(pooled_new, inp_kpool->new_pool_rep));
|
|
ggml_tensor * pooled = kpool_cache.gather_pooled(inp_kpool->pool_cells);
|
|
pooled = ggml_reshape_3d(ctx0, pooled, idx_dim, 1, n_pool);
|
|
cb(pooled, "indexer_k", il);
|
|
|
|
ggml_tensor * q = build_lora_mm(model.layers[il].index_q_proj, cur);
|
|
q = ggml_reshape_3d(ctx0, q, idx_dim, n_idx_h, n_tokens);
|
|
q = build_norm(q, model.layers[il].index_q_norm, nullptr, LLM_NORM_RMS, il);
|
|
q = ggml_rope_multi(ctx0, q, inp_pos, nullptr,
|
|
n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
ext_factor, attn_factor, beta_fast, beta_slow);
|
|
cb(q, "indexer_q", il);
|
|
|
|
// the reference sums the rectified head scores unweighted, scaled by 1/sqrt(head_dim),
|
|
// which is the lightning indexer with every head weight set to that scale
|
|
ggml_tensor * weights = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_idx_h, n_tokens), 1.0f/sqrtf((float) idx_dim));
|
|
ggml_tensor * score = ggml_lightning_indexer(ctx0, q, pooled, weights, inp_kpool->pool_mask); // [n_pool, n_tokens]
|
|
res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, score, il});
|
|
cb(score, "indexer_score", il);
|
|
|
|
const int64_t n_top_pool = std::min<int64_t>(n_pool, hparams.indexer_top_k / kpool);
|
|
ggml_tensor * top_k = ggml_top_k(ctx0, score, n_top_pool); // [n_top_pool, n_tokens], unordered
|
|
cb(top_k, "indexer_top_k", il);
|
|
|
|
// flash attention requires an f16 mask
|
|
const ggml_type type_mask = cparams.flash_attn ? GGML_TYPE_F16 : GGML_TYPE_F32;
|
|
|
|
// keep the cells of the live picked pools and the tail, -inf elsewhere
|
|
// all retained cells are in the token's sequence, so the kq_mask is not needed
|
|
if (cparams.fused_top_k_mask) {
|
|
GGML_ASSERT(kpool*n_top_pool + inp_kpool->tail_idxs->ne[0] == inp_kpool->n_sel);
|
|
|
|
ggml_tensor * keep = ggml_fill(ctx0, ggml_new_tensor_1d(ctx0, type_mask, inp_kpool->n_kv), 0.0f);
|
|
ggml_tensor * sel = ggml_top_k_mask(ctx0, keep, top_k, score, inp_kpool->pool_idxs, inp_kpool->tail_idxs); // [n_kv, n_tokens]
|
|
res->add_fused_node({LLM_FUSED_OP_TOP_K_MASK, sel, il});
|
|
cb(sel, "indexer_sel", il);
|
|
|
|
return sel;
|
|
}
|
|
|
|
// the top blocks, then the incomplete tail with n_kv for missing cells
|
|
ggml_tensor * sel_idx = ggml_get_rows(ctx0, inp_kpool->pool_idxs,
|
|
ggml_reshape_1d(ctx0, top_k, n_top_pool*n_tokens)); // [kpool, n_top_pool*n_tokens]
|
|
sel_idx = ggml_reshape_2d(ctx0, sel_idx, kpool*n_top_pool, n_tokens);
|
|
sel_idx = ggml_concat(ctx0, sel_idx, inp_kpool->tail_idxs, 0);
|
|
const int64_t n_sel = sel_idx->ne[0];
|
|
GGML_ASSERT(n_sel == inp_kpool->n_sel);
|
|
|
|
ggml_build_forward_expand(gf, sel_idx);
|
|
|
|
// TODO: figure out to reduce the large copmute buffer that this creates
|
|
|
|
// scatter zeros for the selected cells into an all -inf row, each dead slot into its own dump row n_kv + slot
|
|
// seeding from sel_idx ties the scatter storage lifetime to this layer
|
|
const int64_t n_kv = inp_kpool->n_kv;
|
|
|
|
ggml_tensor * mask_all = ggml_new_tensor_4d(ctx0, type_mask, n_kv + n_sel, 1, 1, 1);
|
|
mask_all = ggml_fill(ctx0, mask_all, -INFINITY);
|
|
mask_all = ggml_repeat_4d(ctx0, mask_all, n_kv + n_sel, n_tokens, 1, 1);
|
|
mask_all = ggml_reshape_3d(ctx0, mask_all, 1, n_kv + n_sel, n_tokens);
|
|
|
|
ggml_tensor * zeros = ggml_new_tensor_4d(ctx0, type_mask, n_sel, 1, 1, 1);
|
|
zeros = ggml_fill(ctx0, zeros, 0.0f);
|
|
zeros = ggml_repeat_4d(ctx0, zeros, n_sel, n_tokens, 1, 1);
|
|
zeros = ggml_reshape_3d(ctx0, zeros, 1, n_sel, n_tokens);
|
|
|
|
// live slots address disjoint cells, but padded pools and missing tail cells share the n_kv sentinel, and
|
|
// top_k fills a short selection with invisible pools that can overlap the tail, so the scatter would write
|
|
// some cells from several threads: map every dead slot to its own dump row, idx = dump + live*(idx - dump)
|
|
// a picked pool is live when visible: a visible score is a rectified sum >= 0, an invisible one is -inf
|
|
ggml_tensor * top_score = ggml_get_rows(ctx0, ggml_reshape_3d(ctx0, score, 1, n_pool, n_tokens), top_k); // [1, n_top_pool, n_tokens]
|
|
ggml_tensor * live_pool = ggml_clamp(ctx0, ggml_scale_bias(ctx0, top_score, 1.0f, 1.0f), 0.0f, 1.0f);
|
|
live_pool = ggml_reshape_2d(ctx0, ggml_repeat_4d(ctx0, live_pool, kpool, n_top_pool, n_tokens, 1), kpool*n_top_pool, n_tokens);
|
|
// a tail cell is live unless it is the n_kv sentinel
|
|
ggml_tensor * live_tail = ggml_cast(ctx0, inp_kpool->tail_idxs, GGML_TYPE_F32);
|
|
live_tail = ggml_clamp(ctx0, ggml_scale_bias(ctx0, live_tail, -1.0f, (float) n_kv), 0.0f, 1.0f);
|
|
ggml_tensor * live = ggml_concat(ctx0, live_pool, live_tail, 0); // [n_sel, n_tokens]
|
|
|
|
// dump rows n_kv + slot as a cumulative sum: the meta backend cannot split an arange, which has no source
|
|
ggml_tensor * dump = ggml_scale_bias(ctx0, ggml_cumsum(ctx0, ggml_fill(ctx0, live, 1.0f)), 1.0f, (float) (n_kv - 1));
|
|
ggml_tensor * idx_f = ggml_cast(ctx0, sel_idx, GGML_TYPE_F32);
|
|
idx_f = ggml_add(ctx0, ggml_mul(ctx0, ggml_sub(ctx0, idx_f, dump), live), dump);
|
|
sel_idx = ggml_cast(ctx0, idx_f, GGML_TYPE_I32);
|
|
|
|
ggml_tensor * sel = ggml_set_rows(ctx0, mask_all, zeros, ggml_reshape_3d(ctx0, sel_idx, n_sel, n_tokens, 1));
|
|
|
|
sel = ggml_view_2d(ctx0, sel, n_kv, n_tokens, sel->nb[2], 0);
|
|
sel = ggml_cont(ctx0, sel);
|
|
cb(sel, "indexer_sel", il);
|
|
|
|
return sel;
|
|
}
|
|
|
|
// Dense GQA self-attention over the cells that the QSA mask keeps.
|
|
ggml_tensor * llama_model_qwen4exp::graph::build_attn_qsa(
|
|
llm_graph_input_attn_kv * inp,
|
|
ggml_tensor * q_cur,
|
|
ggml_tensor * k_cur,
|
|
ggml_tensor * v_cur,
|
|
ggml_tensor * sel,
|
|
int64_t n_sel,
|
|
float kq_scale,
|
|
int il) {
|
|
// rotate q/k/v before they reach a quantized cache, as the dense path does. the indexer
|
|
// has already scored with its own query in build_qsa_sel, so the selection is unaffected.
|
|
if (inp->self_k_rot[il]) {
|
|
q_cur = llama_mul_mat_hadamard(ctx0, q_cur, inp->self_k_rot[il]);
|
|
k_cur = llama_mul_mat_hadamard(ctx0, k_cur, inp->self_k_rot[il]);
|
|
}
|
|
|
|
if (inp->self_v_rot[il]) {
|
|
v_cur = llama_mul_mat_hadamard(ctx0, v_cur, inp->self_v_rot[il]);
|
|
}
|
|
|
|
// these nodes are added to the graph together so that they are not reordered
|
|
// by doing so, the number of splits in the graph is reduced
|
|
// expand k later to enable rope fusion which directly writes into k-v cache
|
|
ggml_build_forward_expand(gf, q_cur);
|
|
ggml_build_forward_expand(gf, v_cur);
|
|
ggml_build_forward_expand(gf, k_cur);
|
|
|
|
const auto * mctx_cur = inp->mctx;
|
|
|
|
// store to KV cache
|
|
{
|
|
const auto & k_idxs = inp->get_k_idxs();
|
|
const auto & v_idxs = inp->get_v_idxs();
|
|
|
|
ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, k_cur, k_idxs, il));
|
|
ggml_build_forward_expand(gf, mctx_cur->cpy_v(ctx0, v_cur, v_idxs, il));
|
|
}
|
|
|
|
ggml_tensor * q = q_cur;
|
|
ggml_tensor * k = mctx_cur->get_k(ctx0, il);
|
|
ggml_tensor * v = mctx_cur->get_v(ctx0, il);
|
|
|
|
// the selection replaces the kq_mask: split it into the streams of the K view
|
|
const int64_t n_stream = k->ne[3];
|
|
GGML_ASSERT(sel->ne[0] == k->ne[2] && sel->ne[1] % n_stream == 0);
|
|
sel = ggml_reshape_4d(ctx0, sel, sel->ne[0], sel->ne[1]/n_stream, 1, n_stream);
|
|
cb(sel, "kq_mask_qsa", il);
|
|
|
|
ggml_tensor * cur = build_attn_mha(q, k, v, nullptr, sel, nullptr, nullptr, nullptr, n_sel, kq_scale, il);
|
|
cb(cur, "kqv_out", il);
|
|
|
|
// the rotation is its own inverse, so undo it on the value side of the output
|
|
if (inp->self_v_rot[il]) {
|
|
cur = llama_mul_mat_hadamard(ctx0, cur, inp->self_v_rot[il]);
|
|
}
|
|
|
|
return cur;
|
|
}
|
|
|
|
ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn(
|
|
llm_graph_input_attn_kv * inp,
|
|
const llama_memory_hybrid_idx_context * mctx_hyb,
|
|
llm_graph_input_kpool * inp_kpool,
|
|
ggml_tensor * cur,
|
|
ggml_tensor * inp_pos,
|
|
int * sections,
|
|
int il) {
|
|
const int64_t n_embd_head = hparams.n_embd_head_v();
|
|
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
|
|
|
|
// indexer reads the same block input as q/k/v; no cache or no ratio means dense
|
|
const bool qsa = inp_kpool != nullptr && hparams.dsv4_compress_ratios[il] > 0;
|
|
|
|
ggml_tensor * sel = qsa ? build_qsa_sel(mctx_hyb, inp_kpool, cur, inp_pos, sections, il) : nullptr;
|
|
|
|
// Qwen3Next uses a single Q projection that outputs query + gate
|
|
ggml_tensor * Qcur_full = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s); // [ (n_embd_head * 2) * n_head, n_tokens ]
|
|
cb(Qcur_full, "Qcur_full", il);
|
|
|
|
ggml_tensor * Qcur = ggml_view_3d(ctx0, Qcur_full, n_embd_head, n_head, n_tokens,
|
|
ggml_element_size(Qcur_full) * n_embd_head * 2,
|
|
ggml_element_size(Qcur_full) * n_embd_head * 2 * n_head, 0);
|
|
cb(Qcur, "Qcur_reshaped", il);
|
|
|
|
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il);
|
|
cb(Qcur, "Qcur_normed", il);
|
|
|
|
ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur, model.layers[il].wk_s);
|
|
cb(Kcur, "Kcur", il);
|
|
|
|
ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur, model.layers[il].wv_s);
|
|
cb(Vcur, "Vcur", il);
|
|
|
|
Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
|
|
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il);
|
|
cb(Kcur, "Kcur_normed", il);
|
|
|
|
ggml_tensor * gate = ggml_view_3d(ctx0, Qcur_full, n_embd_head, n_head, n_tokens,
|
|
ggml_element_size(Qcur_full) * n_embd_head * 2,
|
|
ggml_element_size(Qcur_full) * n_embd_head * 2 * n_head,
|
|
ggml_element_size(Qcur_full) * n_embd_head);
|
|
gate = ggml_cont_2d(ctx0, gate, n_embd_head * n_head, n_tokens);
|
|
cb(gate, "gate_reshaped", il);
|
|
|
|
Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
|
|
|
|
// Apply IMRoPE
|
|
Qcur = ggml_rope_multi(
|
|
ctx0, Qcur, inp_pos, nullptr,
|
|
n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
);
|
|
|
|
Kcur = ggml_rope_multi(
|
|
ctx0, Kcur, inp_pos, nullptr,
|
|
n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
);
|
|
|
|
cb(Qcur, "Qcur", il);
|
|
cb(Kcur, "Kcur", il);
|
|
cb(Vcur, "Vcur", il);
|
|
|
|
const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
|
|
|
|
if (sel) {
|
|
cur = build_attn_qsa(inp, Qcur, Kcur, Vcur, sel, inp_kpool->n_sel, kq_scale, il);
|
|
} else {
|
|
cur = build_attn(inp,
|
|
nullptr, nullptr, nullptr,
|
|
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
|
|
}
|
|
cb(cur, "attn_pregate", il);
|
|
|
|
ggml_tensor * gate_sigmoid = ggml_sigmoid(ctx0, gate);
|
|
cb(gate_sigmoid, "gate_sigmoid", il);
|
|
|
|
cur = ggml_mul(ctx0, cur, gate_sigmoid);
|
|
cb(cur, "attn_gated", il);
|
|
|
|
cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s);
|
|
cb(cur, "attn_output", il);
|
|
|
|
return cur;
|
|
}
|
|
|
|
ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn_linear(
|
|
llm_graph_input_rs * inp,
|
|
ggml_tensor * cur,
|
|
int il) {
|
|
const auto * mctx_cur = inp->mctx;
|
|
|
|
const int64_t d_inner = hparams.ssm_d_inner;
|
|
const int64_t n_seqs = ubatch.n_seqs;
|
|
const int64_t head_k_dim = hparams.ssm_d_state;
|
|
const int64_t num_k_heads = hparams.ssm_n_group;
|
|
const int64_t num_v_heads = hparams.ssm_dt_rank;
|
|
const int64_t head_v_dim = hparams.ssm_d_state;
|
|
const int64_t n_seq_tokens = ubatch.n_seq_tokens;
|
|
|
|
GGML_ASSERT(n_seqs != 0);
|
|
GGML_ASSERT(ubatch.equal_seqs());
|
|
GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);
|
|
GGML_ASSERT(head_v_dim * num_v_heads == d_inner);
|
|
|
|
auto qkvz = build_qkvz(cur, il);
|
|
ggml_tensor * qkv_mixed = qkvz.first;
|
|
ggml_tensor * z = qkvz.second;
|
|
|
|
ggml_tensor * beta = nullptr;
|
|
ggml_tensor * alpha = nullptr;
|
|
if (model.layers[il].ssm_beta_alpha_cat) {
|
|
// beta and alpha read the same input: one matmul, then the beta rows and the alpha rows
|
|
ggml_tensor * ba = build_lora_mm(model.layers[il].ssm_beta_alpha_cat, cur);
|
|
cb(ba, "beta_alpha", il);
|
|
|
|
beta = ggml_view_4d(ctx0, ba, 1, num_v_heads, n_seq_tokens, n_seqs,
|
|
ggml_element_size(ba), ba->nb[1], ba->nb[1] * n_seq_tokens, 0);
|
|
alpha = ggml_view_3d(ctx0, ba, num_v_heads, n_seq_tokens, n_seqs,
|
|
ba->nb[1], ba->nb[1] * n_seq_tokens, ggml_row_size(ba->type, num_v_heads));
|
|
} else {
|
|
beta = build_lora_mm(model.layers[il].ssm_beta, cur, model.layers[il].ssm_beta_s);
|
|
beta = ggml_reshape_4d(ctx0, beta, 1, num_v_heads, n_seq_tokens, n_seqs);
|
|
|
|
alpha = build_lora_mm(model.layers[il].ssm_alpha, cur, model.layers[il].ssm_alpha_s);
|
|
alpha = ggml_reshape_3d(ctx0, alpha, num_v_heads, n_seq_tokens, n_seqs);
|
|
}
|
|
cb(beta, "beta", il);
|
|
cb(alpha, "alpha", il);
|
|
|
|
beta = ggml_sigmoid(ctx0, beta);
|
|
cb(beta, "beta_sigmoid", il);
|
|
|
|
ggml_tensor * alpha_biased = ggml_add(ctx0, alpha, model.layers[il].ssm_dt);
|
|
ggml_tensor * alpha_softplus = ggml_softplus(ctx0, alpha_biased);
|
|
cb(alpha_softplus, "a_softplus", il);
|
|
|
|
ggml_tensor * gate = ggml_mul(ctx0, alpha_softplus, model.layers[il].ssm_a); // -A_log.exp() * softplus
|
|
cb(gate, "gate", il);
|
|
|
|
gate = ggml_reshape_4d(ctx0, gate, 1, num_v_heads, n_seq_tokens, n_seqs);
|
|
|
|
ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);
|
|
ggml_tensor * ssm_states_all = mctx_cur->get_s_l(il);
|
|
|
|
ggml_tensor * conv_kernel = model.layers[il].ssm_conv1d;
|
|
const int64_t conv_kernel_size = conv_kernel->ne[0];
|
|
|
|
// the channels must match how load_arch_tensors sizes wqkv, not ssm_d_inner
|
|
const int64_t conv_channels = head_k_dim * num_k_heads * 2 + head_v_dim * num_v_heads;
|
|
|
|
ggml_tensor * conv_input = build_conv_state_at(inp, conv_states_all, qkv_mixed,
|
|
conv_kernel_size - 1, conv_channels, il);
|
|
|
|
ggml_tensor * state = build_rs(inp, ssm_states_all, hparams.n_embd_s(), n_seqs);
|
|
state = ggml_reshape_4d(ctx0, state, head_v_dim, head_v_dim, num_v_heads, n_seqs);
|
|
cb(state, "state_predelta", il);
|
|
|
|
ggml_tensor * conv_output_proper = ggml_ssm_conv(ctx0, conv_input, conv_kernel);
|
|
cb(conv_output_proper, "conv_output_raw", il);
|
|
|
|
ggml_tensor * conv_output_silu = ggml_silu(ctx0, conv_output_proper);
|
|
cb(conv_output_silu, "conv_output_silu", il);
|
|
|
|
ggml_tensor * conv_qkv_mix = conv_output_silu;
|
|
|
|
int64_t nb1_qkv = ggml_row_size(conv_qkv_mix->type, conv_channels);
|
|
|
|
// Extract the convolved Q, K, V from conv_output
|
|
ggml_tensor * q_conv = ggml_view_4d(ctx0, conv_qkv_mix, head_k_dim, num_k_heads, n_seq_tokens, n_seqs,
|
|
ggml_row_size(conv_qkv_mix->type, head_k_dim),
|
|
nb1_qkv,
|
|
nb1_qkv * n_seq_tokens,
|
|
0);
|
|
|
|
ggml_tensor * k_conv = ggml_view_4d(ctx0, conv_qkv_mix, head_k_dim, num_k_heads, n_seq_tokens, n_seqs,
|
|
ggml_row_size(conv_qkv_mix->type, head_k_dim),
|
|
nb1_qkv,
|
|
nb1_qkv * n_seq_tokens,
|
|
head_k_dim * num_k_heads * ggml_element_size(conv_qkv_mix));
|
|
|
|
ggml_tensor * v_conv = ggml_view_4d(ctx0, conv_qkv_mix, head_v_dim, num_v_heads, n_seq_tokens, n_seqs,
|
|
ggml_row_size(conv_qkv_mix->type, head_v_dim),
|
|
nb1_qkv,
|
|
nb1_qkv * n_seq_tokens,
|
|
ggml_row_size(conv_qkv_mix->type, 2 * head_k_dim * num_k_heads));
|
|
|
|
cb(q_conv, "q_conv", il);
|
|
cb(k_conv, "k_conv", il);
|
|
cb(v_conv, "v_conv", il);
|
|
|
|
|
|
const float eps_norm = hparams.f_norm_rms_eps;
|
|
|
|
q_conv = build_gdn_l2_norm(ctx0, q_conv, eps_norm);
|
|
k_conv = build_gdn_l2_norm(ctx0, k_conv, eps_norm);
|
|
|
|
// repeat to match shapes when head keys != value keys; unneeded with the fused GDN
|
|
if (num_k_heads != num_v_heads && (!cparams.fused_gdn_ar || !cparams.fused_gdn_ch)) {
|
|
GGML_ASSERT(num_v_heads % num_k_heads == 0);
|
|
q_conv = ggml_repeat_4d(ctx0, q_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs);
|
|
k_conv = ggml_repeat_4d(ctx0, k_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs);
|
|
}
|
|
|
|
cb(q_conv, "q_conv_predelta", il);
|
|
cb(k_conv, "k_conv_predelta", il);
|
|
cb(v_conv, "v_conv_predelta", il);
|
|
|
|
ggml_tensor * output = build_recurrent_attn(inp, ssm_states_all, q_conv, k_conv, v_conv, gate, beta, state, il);
|
|
|
|
ggml_tensor * z_2d = ggml_view_4d(ctx0, z, head_v_dim, num_v_heads, n_seq_tokens, n_seqs,
|
|
ggml_row_size(z->type, head_v_dim), z->nb[1], z->nb[1] * n_seq_tokens, 0);
|
|
|
|
// gated normalization, as self.norm(core_attn_out, z) in the reference
|
|
ggml_tensor * attn_out_norm = build_norm_gated(output, model.layers[il].ssm_norm, z_2d, il);
|
|
|
|
ggml_tensor * final_output = ggml_reshape_2d(ctx0, attn_out_norm, head_v_dim * num_v_heads, n_seq_tokens * n_seqs);
|
|
cb(final_output, "final_output", il);
|
|
|
|
cur = build_lora_mm(model.layers[il].ssm_out, final_output, model.layers[il].ssm_out_s);
|
|
cb(cur, "linear_attn_out", il);
|
|
|
|
return cur;
|
|
}
|
|
|
|
ggml_tensor * llama_model_qwen4exp::graph::build_layer_ffn(ggml_tensor * cur, const int il) {
|
|
GGML_ASSERT(model.layers[il].ffn_gate_inp != nullptr);
|
|
|
|
ggml_tensor * moe_out =
|
|
build_moe_ffn(cur,
|
|
model.layers[il].ffn_gate_inp,
|
|
model.layers[il].ffn_up_exps,
|
|
model.layers[il].ffn_gate_exps,
|
|
model.layers[il].ffn_down_exps,
|
|
nullptr,
|
|
n_expert, n_expert_used,
|
|
LLM_FFN_SILU, true,
|
|
hparams.expert_weights_scale,
|
|
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
|
|
nullptr, model.layers[il].ffn_gate_up_exps,
|
|
model.layers[il].ffn_up_exps_s,
|
|
model.layers[il].ffn_gate_exps_s,
|
|
model.layers[il].ffn_down_exps_s);
|
|
cb(moe_out, "ffn_moe_out", il);
|
|
|
|
// shared experts, as in the Qwen3Next reference
|
|
if (model.layers[il].ffn_up_shexp != nullptr) {
|
|
ggml_tensor * ffn_shexp =
|
|
build_ffn(cur,
|
|
model.layers[il].ffn_up_shexp, NULL, model.layers[il].ffn_up_shexp_s,
|
|
model.layers[il].ffn_gate_shexp, NULL, model.layers[il].ffn_gate_shexp_s,
|
|
model.layers[il].ffn_down_shexp, NULL, model.layers[il].ffn_down_shexp_s,
|
|
NULL,
|
|
LLM_FFN_SILU, LLM_FFN_PAR, il);
|
|
cb(ffn_shexp, "ffn_shexp", il);
|
|
|
|
// shared expert has its own sigmoided gate (ffn_gate_inp_shexp, one value per token)
|
|
ggml_tensor * shared_gate = build_lora_mm(model.layers[il].ffn_gate_inp_shexp, cur);
|
|
cb(shared_gate, "shared_expert_gate", il);
|
|
|
|
shared_gate = ggml_sigmoid(ctx0, shared_gate);
|
|
cb(shared_gate, "shared_expert_gate_sigmoid", il);
|
|
|
|
ffn_shexp = ggml_mul(ctx0, ffn_shexp, shared_gate);
|
|
cb(ffn_shexp, "ffn_shexp_gated", il);
|
|
|
|
cur = ggml_add(ctx0, moe_out, ffn_shexp);
|
|
cb(cur, "ffn_out", il);
|
|
} else {
|
|
cur = moe_out;
|
|
}
|
|
|
|
return cur;
|
|
}
|
|
|
|
// PLE n-gram hash embedding: each token gathers ple_n_heads rows of a shared table.
|
|
// mixed_n = (t[p]*m[0]) ^ ... ^ (t[p-n+1]*m[n-1]); row = mixed_n % vocab[h] + offset[h]
|
|
// The hash runs host-side because ggml has no int64 and no xor. EOS resets the window.
|
|
|
|
class llm_graph_input_qwen4exp_ple : public llm_graph_input_i {
|
|
public:
|
|
llm_graph_input_qwen4exp_ple(const llama_model & model,
|
|
const llama_kv_cache_context * mctx) : model(model), mctx(mctx) {}
|
|
virtual ~llm_graph_input_qwen4exp_ple() = default;
|
|
|
|
void set_input(const llama_ubatch * ubatch) override;
|
|
|
|
bool can_reuse(const llm_graph_params & params) override {
|
|
mctx = static_cast<const llama_memory_hybrid_idx_context *>(params.mctx)->get_attn();
|
|
return rows->ne[0] == (int64_t) model.hparams.ple_n_heads * params.ubatch.n_tokens;
|
|
}
|
|
|
|
ggml_tensor * rows = nullptr; // I32 [ple_n_heads * n_tokens]
|
|
|
|
const llama_model & model;
|
|
|
|
// the predecessor tokens live in the attention KV cells (ext.tok)
|
|
const llama_kv_cache_context * mctx;
|
|
|
|
// scratch, reused across set_input() calls
|
|
std::vector<llama_token> prev;
|
|
};
|
|
|
|
void llm_graph_input_qwen4exp_ple::set_input(const llama_ubatch * ubatch) {
|
|
const auto & hparams = model.hparams;
|
|
|
|
// an image arrives as an embd batch, so ubatch->token is null, but every position still needs a row for ggml_get_rows
|
|
// stand in the image token id that the reference hashes, or EOS if the file has no such key
|
|
// gemma3n and gemma4 do the same with a hardcoded row 0 of per_layer_token_embd.
|
|
const llama_token img_tok = hparams.ple_image_token_id != 0
|
|
? (llama_token) hparams.ple_image_token_id
|
|
: (llama_token) hparams.ple_eos_token_id;
|
|
auto tok_of = [&](int64_t k) -> llama_token {
|
|
const bool is_embd = !ubatch->token || (ubatch->is_mixed() && ubatch->type[k]);
|
|
return is_embd ? img_tok : ubatch->token[k];
|
|
};
|
|
|
|
const int64_t n_tokens = ubatch->n_tokens;
|
|
const int64_t n_gram = hparams.ple_ngram_size;
|
|
const int64_t n_heads = hparams.ple_n_heads;
|
|
const int64_t per_gram = hparams.ple_heads_per_ngram;
|
|
const int64_t eos = hparams.ple_eos_token_id;
|
|
const int64_t n_prev = n_gram - 1;
|
|
|
|
std::vector<int32_t> idx(n_heads * n_tokens);
|
|
|
|
GGML_ASSERT(mctx != nullptr);
|
|
|
|
for (int64_t i = 0; i < n_tokens; ++i) {
|
|
// the preceding tokens would be ambiguous, see get_prev_tokens()
|
|
GGML_ASSERT(ubatch->n_seq_id[i] == 1 && "PLE n-gram embeddings do not support tokens shared by multiple sequences");
|
|
}
|
|
|
|
// predecessors come from the KV cells (ext.tok); apply_ubatch() already stored this ubatch, so its own tokens count too
|
|
mctx->get_prev_tokens(*ubatch, n_prev, prev);
|
|
|
|
for (int64_t i = 0; i < n_tokens; ++i) {
|
|
// an EOS in the window resets everything at or before it
|
|
// a missing predecessor (before the sequence start, or no cached cell) reads as EOS
|
|
// the EOS of the token itself does not cut its own context, as in the reference
|
|
std::vector<int64_t> ctx(n_gram);
|
|
ctx[0] = tok_of(i);
|
|
bool cut = false;
|
|
for (int64_t s = 1; s < n_gram; ++s) {
|
|
// predecessor s positions back; prev[] is oldest-first, missing entries are LLAMA_TOKEN_NULL
|
|
const llama_token t = cut ? LLAMA_TOKEN_NULL : prev[i*n_prev + (n_prev - s)];
|
|
cut = cut || t < 0 || t == eos;
|
|
ctx[s] = cut ? eos : t;
|
|
}
|
|
|
|
for (int64_t n = 2; n <= n_gram; ++n) {
|
|
uint64_t mixed = (uint64_t) ctx[0] * hparams.ple_layer_multipliers[0];
|
|
for (int64_t j = 1; j < n; ++j) {
|
|
mixed ^= (uint64_t) ctx[j] * hparams.ple_layer_multipliers[j];
|
|
}
|
|
const int64_t base = (n - 2) * per_gram;
|
|
for (int64_t g = 0; g < per_gram; ++g) {
|
|
const int64_t h_i = base + g;
|
|
idx[i * n_heads + h_i] =
|
|
(int32_t) (mixed % hparams.ple_head_vocab_sizes[h_i] + hparams.ple_head_offsets[h_i]);
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
ggml_tensor * ple = model.per_layer_tok_embd;
|
|
|
|
const bool prefetch = model.can_prefetch.count(ple);
|
|
if (prefetch) {
|
|
llama_prefetch_rows(ple, idx.data(), idx.size());
|
|
}
|
|
}
|
|
|
|
ggml_backend_tensor_set(rows, idx.data(), 0, idx.size()*ggml_element_size(rows));
|
|
}
|
|
|
|
// Read a conv history out of its own recurrent row and write the new tail back.
|
|
// The shared build_conv_state cannot do this: qwen4exp has two such rows per layer.
|
|
ggml_tensor * llama_model_qwen4exp::graph::build_conv_state_at(
|
|
llm_graph_input_rs * inp,
|
|
ggml_tensor * conv_states_all,
|
|
ggml_tensor * x,
|
|
int64_t state_cols,
|
|
int64_t channels,
|
|
int il) {
|
|
const auto * mctx_cur = inp->mctx;
|
|
|
|
const auto kv_head = mctx_cur->get_head();
|
|
|
|
const int64_t n_seqs = ubatch.n_seqs;
|
|
const int64_t row_total = conv_states_all->ne[0];
|
|
|
|
// the row is exactly this convolution's state, so the gather is reused as a whole
|
|
GGML_ASSERT(state_cols * channels == row_total);
|
|
|
|
auto it = rs_rows.find(conv_states_all);
|
|
if (it == rs_rows.end()) {
|
|
it = rs_rows.emplace(conv_states_all, build_rs(inp, conv_states_all, row_total, n_seqs)).first;
|
|
}
|
|
ggml_tensor * rows = it->second;
|
|
|
|
ggml_tensor * state = ggml_reshape_3d(ctx0, rows, state_cols, channels, n_seqs);
|
|
cb(state, "conv_state_at", il);
|
|
|
|
ggml_tensor * conv_input = ggml_concat(ctx0, state, ggml_transpose(ctx0, x), 0);
|
|
|
|
// [TAG_RECURRENT_ROLLBACK_SPLITS] keep the last state_cols columns once per rollback slot,
|
|
// slot s ending s tokens earlier so a rollback of s tokens reads a history that never saw them
|
|
const size_t row_size = ggml_row_size(conv_states_all->type, row_total);
|
|
const uint32_t mem_size = mctx_cur->get_size();
|
|
|
|
const int64_t n_slots = (int64_t) cparams.n_rs_seq + 1;
|
|
|
|
for (int64_t slot = 0; slot < n_slots; ++slot) {
|
|
const int64_t s_idx = std::max<int64_t>(0, conv_input->ne[0] - state_cols - slot);
|
|
|
|
ggml_tensor * tail = ggml_view_3d(ctx0, conv_input,
|
|
state_cols, channels, n_seqs,
|
|
conv_input->nb[1], conv_input->nb[2],
|
|
ggml_row_size(conv_input->type, s_idx));
|
|
|
|
ggml_tensor * dst = ggml_view_2d(ctx0, conv_states_all,
|
|
state_cols * channels, n_seqs,
|
|
conv_states_all->nb[1],
|
|
(slot * mem_size + kv_head) * row_size);
|
|
|
|
ggml_build_forward_expand(gf, ggml_cpy(ctx0, tail, dst));
|
|
}
|
|
|
|
return conv_input;
|
|
}
|
|
|
|
ggml_tensor * llama_model_qwen4exp::graph::build_inp_ple(
|
|
const llama_memory_hybrid_idx_context * mctx_hyb) {
|
|
const int64_t n_heads = hparams.ple_n_heads;
|
|
|
|
// the attention cells see every ubatch regardless of the layer types
|
|
auto ple_inp = std::make_unique<llm_graph_input_qwen4exp_ple>(model, mctx_hyb->get_attn());
|
|
|
|
ple_inp->rows = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_heads * n_tokens);
|
|
ggml_set_input(ple_inp->rows);
|
|
ggml_tensor * rows = ple_inp->rows;
|
|
res->add_input(std::move(ple_inp));
|
|
|
|
// gather then flatten the heads: get_rows lays the head dimension out slowest, as the reference does
|
|
ggml_tensor * emb = ggml_get_rows(ctx0, model.per_layer_tok_embd, rows);
|
|
emb = ggml_reshape_2d(ctx0, emb, hparams.ple_head_dim * n_heads, n_tokens);
|
|
cb(emb, "ple_embd", -1);
|
|
|
|
return emb;
|
|
}
|
|
|
|
ggml_tensor * llama_model_qwen4exp::graph::build_ple(
|
|
llm_graph_input_rs * inp,
|
|
ggml_tensor * emb,
|
|
ggml_tensor * hidden,
|
|
int il) {
|
|
const int64_t hc = hparams.dsv4_hc_mult;
|
|
const int64_t hc_dim = hc * n_embd;
|
|
|
|
ggml_tensor * key = build_lora_mm(model.layers[il].ple_key, emb);
|
|
ggml_tensor * value = build_lora_mm(model.layers[il].ple_value, emb);
|
|
|
|
// both norms group over one hc stream, with a [n_embd, hc] weight
|
|
auto grouped_norm = [&](ggml_tensor * x, ggml_tensor * w) {
|
|
ggml_tensor * t = ggml_reshape_3d(ctx0, x, n_embd, hc, n_tokens);
|
|
return ggml_mul(ctx0, ggml_rms_norm(ctx0, t, hparams.f_norm_rms_eps), w);
|
|
};
|
|
|
|
key = grouped_norm(key, model.layers[il].ple_norm_key);
|
|
ggml_tensor * query = grouped_norm(hidden, model.layers[il].ple_norm_query);
|
|
|
|
// per-stream dot product, then a signed square root before the sigmoid
|
|
ggml_tensor * s = ggml_sum_rows(ctx0, ggml_mul(ctx0, key, query));
|
|
s = ggml_scale(ctx0, s, 1.0f / sqrtf((float) n_embd));
|
|
|
|
ggml_tensor * mag = ggml_sqrt(ctx0, ggml_clamp(ctx0, ggml_abs(ctx0, s), 1e-6f, INFINITY));
|
|
ggml_tensor * gate = ggml_sigmoid(ctx0, ggml_mul(ctx0, ggml_sgn(ctx0, s), mag));
|
|
cb(gate, "ple_gate", il);
|
|
|
|
// [n_embd, 1, T] value broadcast across the hc streams, scaled by the gate
|
|
ggml_tensor * v3 = ggml_reshape_3d(ctx0, value, n_embd, 1, n_tokens);
|
|
v3 = ggml_repeat_4d(ctx0, v3, n_embd, hc, n_tokens, 1);
|
|
|
|
ggml_tensor * gated = ggml_mul(ctx0, v3, gate);
|
|
cb(gated, "ple_gated_value", il);
|
|
|
|
ggml_tensor * normalized = grouped_norm(
|
|
ggml_reshape_2d(ctx0, gated, hc_dim, n_tokens),
|
|
model.layers[il].ple_norm_conv);
|
|
normalized = ggml_reshape_2d(ctx0, normalized, hc_dim, n_tokens);
|
|
|
|
// depthwise causal conv, dilated by the n-gram size, as a sum of shifted copies
|
|
// ggml_conv_1d_dw is documented as unreliable:
|
|
// out[c, t] = sum_k w[k, c] * x[c, t - (K-1-k)*dilation]
|
|
// The history of the earlier ubatches is prepended, so a chunked prefill matches a single-shot one.
|
|
const int64_t kern = hparams.ple_conv_kernel;
|
|
const int64_t dil = hparams.ple_ngram_size;
|
|
const int64_t hist = (kern - 1) * dil;
|
|
|
|
// the conv history is per sequence, so the input carries the sequence axis too
|
|
const int64_t n_seqs = ubatch.n_seqs;
|
|
const int64_t n_seq_tokens = ubatch.n_seq_tokens;
|
|
|
|
// [hist + n_seq_tokens, hc_dim, n_seqs], tokens on ne[0]
|
|
ggml_tensor * padded = build_conv_state_at(inp, inp->mctx->get_p_l(il),
|
|
ggml_reshape_3d(ctx0, normalized, hc_dim, n_seq_tokens, n_seqs),
|
|
hist, hc_dim, il);
|
|
|
|
ggml_tensor * conv_out = nullptr;
|
|
for (int64_t k = 0; k < kern; ++k) {
|
|
// tap k reads (kern-1-k)*dilation positions back
|
|
const int64_t start = hist - (kern - 1 - k) * dil;
|
|
|
|
ggml_tensor * shifted = ggml_cont(ctx0,
|
|
ggml_transpose(ctx0,
|
|
ggml_view_3d(ctx0, padded, n_seq_tokens, hc_dim, n_seqs,
|
|
padded->nb[1], padded->nb[2],
|
|
ggml_row_size(padded->type, start))));
|
|
|
|
// column k of the [kern, hc_dim] kernel is one weight per channel
|
|
ggml_tensor * wk = ggml_cont(ctx0,
|
|
ggml_view_2d(ctx0, model.layers[il].ple_conv1d, 1, hc_dim,
|
|
model.layers[il].ple_conv1d->nb[1],
|
|
k * model.layers[il].ple_conv1d->nb[0]));
|
|
// this kernel keeps the file type, so cast it before it multiplies an f32 activation
|
|
wk = ggml_reshape_1d(ctx0, wk, hc_dim);
|
|
if (wk->type != GGML_TYPE_F32) {
|
|
wk = ggml_cast(ctx0, wk, GGML_TYPE_F32);
|
|
}
|
|
|
|
ggml_tensor * term = ggml_mul(ctx0, shifted, wk);
|
|
conv_out = conv_out ? ggml_add(ctx0, conv_out, term) : term;
|
|
}
|
|
|
|
conv_out = ggml_silu(ctx0, conv_out);
|
|
conv_out = ggml_reshape_3d(ctx0, ggml_cont(ctx0, conv_out), n_embd, hc, n_tokens);
|
|
cb(conv_out, "ple_conv_out", il);
|
|
|
|
return ggml_add(ctx0, hidden, ggml_add(ctx0, gated, conv_out));
|
|
}
|