* WIP OCP FP8 E4M3 support * Fix python type-checker * Materialize input_scales also for FP8 when converting * Fix conversion of https://huggingface.co/RedHatAI/Qwen3.8-27B-NVFP4 1. Per row FP8 wrongly fell to NVFP4 conversion, as NVFP4 checked on dims alone. Also check on dtype for FP8 2. Need to reshape FP8 QKV projections scales in the same way that weights are reshaped * Allow remaining FP8 tensors to be dequantized during conversion * Allow tensor-strategy for FP8-only compressed-tensor ckpts * Fix and test gguf-py behavior for FP8_E4M3 * Disallow QKV-fusion for FP8 temporarily * Fix conversion of fused qkvz & FP8 for Qwen3-Next * Avoid fusing FP8 for --fuse-gate-up-exps as we may have different ws * Emit warnings when not fusing FP8 during conversion * More FP8 conversion fixes * Add supported for strided copies to/from FP8 in CPU * remove unneeded scratch buffer from FP8 cpu cpy/dup path * add explanatory comments and TODOs * Add dedicated encoder and decoder tests for FP8 * Whitelist ggml_op_get_rows in metal instead of blacklisting required to fix failing CI, and white-listing should be more future-proof than adding FP8 to the black-list https://github.com/ggml-org/llama.cpp/actions/runs/38433916498/job/115422880633?pr=27512 * Fix more conversion bugs * Defer weight materialization during conversion * Blacklist fp8 in metal backend instead of white-list * Adjust indents * Fix lint + execute tests * Add fast CPU kernel for F8_E4M3 * Conversion fixes * Fix remaining gaps * Enable tied weights with shared scales * Make mandatory-F32 rules apply to FP8 during conversion * Fix python type checks https://github.com/ggml-org/llama.cpp/actions/runs/39765670052/job/119780379595?pr=27512 * Fix kaleidai unit-test failures https://github.com/ggml-org/llama.cpp/actions/runs/39765670125/job/119780383150 * Relax numeric precision constraints for pre-check * Leave fast-FP8 CPU on RISC-V for follow-up We already exclude arm from the fast-path * Allow output scale tensors to be located in any gguf shard * Move test_conversion_fp8.py test execution to behavior-check.yaml * Apply batched suggestions from code review Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co> * Move mixed compressed-tensor logic into dequant_fp8 + add unit-tests for it * Install all requirements also on Windows for behavior-check Fixes https://github.com/ggml-org/llama.cpp/actions/runs/40535852590/job/122300758223?pr=27512 * Unmap lazy tensors before clean-up in Windows Should fix https://github.com/ggml-org/llama.cpp/actions/runs/40539797824/job/122314782459?pr=27512 * LF line endings https://github.com/ggml-org/llama.cpp/actions/runs/40539798168/job/122314791118?pr=27512 * Extract FP8 tests from llama-arch to its own test-fixture See https://github.com/ggml-org/llama.cpp/pull/27512#discussion_r4400628221 * Update gguf-py/gguf/scripts/gguf_convert_endian.py Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co> --------- Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
813 lines
36 KiB
C++
813 lines
36 KiB
C++
#include "models.h"
|
|
#include "llama-memory-recurrent.h"
|
|
|
|
void llama_model_qwen3next::load_arch_hparams(llama_model_loader & ml) {
|
|
ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false);
|
|
ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);
|
|
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
|
|
|
|
// Load linear attention (gated delta net) parameters
|
|
ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);
|
|
ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner);
|
|
ml.get_key(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state);
|
|
ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
|
|
ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);
|
|
|
|
// Mark recurrent layers (linear attention layers).
|
|
if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer_all, false)) {
|
|
uint32_t full_attn_interval = 4;
|
|
ml.get_key(LLM_KV_FULL_ATTENTION_INTERVAL, full_attn_interval, false);
|
|
for (uint32_t i = 0; i < hparams.n_layer_all; ++i) {
|
|
hparams.is_recr_impl[i] = (i < hparams.n_layer()) && ((i + 1) % full_attn_interval != 0);
|
|
}
|
|
}
|
|
|
|
switch (hparams.n_layer()) {
|
|
case 48: type = LLM_TYPE_80B_A3B; break;
|
|
default: type = LLM_TYPE_UNKNOWN;
|
|
}
|
|
}
|
|
|
|
void llama_model_qwen3next::load_arch_tensors(llama_model_loader & ml) {
|
|
LLAMA_LOAD_LOCALS;
|
|
|
|
if (n_expert == 0) {
|
|
throw std::runtime_error(arch_name() + " model cannot have zero experts");
|
|
}
|
|
|
|
const auto nf = nextn_flags(ml);
|
|
const int trunk_flags = nf.trunk;
|
|
const int mtp_flags = nf.mtp;
|
|
|
|
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0);
|
|
|
|
// output
|
|
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), { n_embd }, 0);
|
|
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED);
|
|
|
|
// if output is NULL, init from the input tok embed
|
|
if (output == NULL) {
|
|
output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, TENSOR_DUPLICATED);
|
|
}
|
|
|
|
const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used;
|
|
|
|
// Calculate dimensions from hyperparameters
|
|
const int64_t head_k_dim = hparams.ssm_d_state;
|
|
const int64_t head_v_dim = hparams.ssm_d_state;
|
|
const int64_t n_k_heads = hparams.ssm_n_group;
|
|
const int64_t n_v_heads = hparams.ssm_dt_rank;
|
|
const int64_t key_dim = head_k_dim * n_k_heads;
|
|
const int64_t value_dim = head_v_dim * n_v_heads;
|
|
const int64_t conv_dim = key_dim * 2 + value_dim;
|
|
|
|
// Calculate projection sizes
|
|
const int64_t qkvz_dim = key_dim * 2 + value_dim * 2;
|
|
const int64_t ba_dim = n_v_heads * 2;
|
|
|
|
auto load_block_trunk = [&](int il, int flags) {
|
|
auto & layer = layers[il];
|
|
const uint32_t n_ff_shexp = hparams.n_ff_shexp > 0 ? hparams.n_ff_shexp : hparams.n_ff(il);
|
|
|
|
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", il), { n_embd }, flags);
|
|
layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", il), { n_embd }, flags);
|
|
|
|
if (!hparams.is_recr(il)) {
|
|
// Attention layers
|
|
create_tensor_qkv(layer, il, n_embd, n_embd_head_k * n_head * 2, n_embd_k_gqa, n_embd_v_gqa, flags);
|
|
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", il), { n_embd_head_k * n_head, n_embd }, flags);
|
|
// Q/K normalization for attention layers
|
|
layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", il), { n_embd_head_k }, flags);
|
|
layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", il), { n_embd_head_k }, flags);
|
|
} else {
|
|
// Linear attention (gated delta net) specific tensors
|
|
// Create tensors with calculated dimensions
|
|
// note: ssm_in is used by legacy GGUF
|
|
layer.ssm_in = create_tensor(tn(LLM_TENSOR_SSM_IN, "weight", il), { n_embd, qkvz_dim }, TENSOR_NOT_REQUIRED | flags);
|
|
layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", il), { n_embd, key_dim * 2 + value_dim }, TENSOR_NOT_REQUIRED | flags);
|
|
layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", il), { n_embd, value_dim }, TENSOR_NOT_REQUIRED | flags);
|
|
layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "weight", il), { hparams.ssm_d_conv, conv_dim }, flags);
|
|
layer.ssm_dt = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", il), { hparams.ssm_dt_rank }, flags);
|
|
layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, il), { hparams.ssm_dt_rank }, flags);
|
|
layer.ssm_beta_alpha = create_tensor(tn(LLM_TENSOR_SSM_BETA_ALPHA, "weight", il), { n_embd, ba_dim }, flags);
|
|
layer.ssm_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", il), { head_v_dim }, flags);
|
|
layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, "weight", il), { value_dim, n_embd }, flags);
|
|
}
|
|
|
|
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", il), { n_embd, n_expert }, flags);
|
|
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), { n_ff_exp, n_embd, n_expert }, flags);
|
|
create_tensor_gate_up_exps(layer, il, n_embd, n_ff_exp, n_expert, flags);
|
|
|
|
// Shared experts
|
|
layer.ffn_gate_inp_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP_SHEXP, "weight", il), { n_embd }, flags);
|
|
layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", il), { n_embd, n_ff_shexp }, flags);
|
|
layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", il), { n_embd, n_ff_shexp }, flags);
|
|
layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", il), { n_ff_shexp, n_embd }, flags);
|
|
};
|
|
|
|
auto load_block_mtp = [&](int il) {
|
|
// MTP head is identical to the trunk block (full attention + FFN)
|
|
load_block_trunk(il, mtp_flags);
|
|
|
|
auto & layer = layers[il];
|
|
|
|
// NextN-specific tensors that define the MTP block.
|
|
layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", il), { 2 * n_embd, n_embd }, mtp_flags);
|
|
layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", il), { n_embd }, mtp_flags);
|
|
layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", il), { n_embd }, mtp_flags);
|
|
layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", il), { n_embd, n_vocab }, mtp_flags | TENSOR_NOT_REQUIRED);
|
|
layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", il), { n_embd, n_vocab }, mtp_flags | TENSOR_NOT_REQUIRED);
|
|
layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", il), { n_embd }, mtp_flags | TENSOR_NOT_REQUIRED);
|
|
};
|
|
|
|
for (int i = 0; i < n_layer; i++) {
|
|
load_block_trunk(i, trunk_flags);
|
|
}
|
|
for (int i = n_layer; i < n_layer_all; i++) {
|
|
load_block_mtp(i);
|
|
}
|
|
}
|
|
|
|
std::unique_ptr<llm_graph_context> llama_model_qwen3next::build_arch_graph(const llm_graph_params & params) const {
|
|
if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) {
|
|
return std::make_unique<graph_mtp>(*this, params);
|
|
}
|
|
return std::make_unique<graph>(*this, params);
|
|
}
|
|
|
|
llama_model_qwen3next::graph::graph(const llama_model & model, const llm_graph_params & params) :
|
|
llm_build_delta_net_base(params), model(model) {
|
|
ggml_tensor * cur;
|
|
ggml_tensor * inpL;
|
|
|
|
inpL = build_inp_embd(model.tok_embd);
|
|
cb(inpL, "model.embed_tokens", -1);
|
|
|
|
auto * inp = build_inp_mem_hybrid();
|
|
|
|
ggml_tensor * inp_pos = build_inp_pos();
|
|
ggml_tensor * inp_out_ids = build_inp_out_ids();
|
|
|
|
// MTP/NextN layers are loaded as extra decoder blocks but not executed in the main pass.
|
|
for (int il = 0; il < n_layer; ++il) {
|
|
res->t_layer_inp[il] = inpL;
|
|
|
|
ggml_tensor * inpSA = inpL;
|
|
|
|
cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il);
|
|
cb(cur, "attn_norm", il);
|
|
|
|
ggml_build_forward_expand(gf, cur);
|
|
|
|
// Determine layer type and build appropriate attention mechanism
|
|
if (hparams.is_recr(il)) {
|
|
// Linear attention layer (gated delta net)
|
|
cur = build_layer_attn_linear(inp->get_recr(), cur, il);
|
|
} else {
|
|
// Full attention layer
|
|
cur = build_layer_attn(inp->get_attn(), cur, inp_pos, il);
|
|
}
|
|
|
|
if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {
|
|
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
|
|
inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
|
|
}
|
|
|
|
// Residual connection
|
|
cur = ggml_add(ctx0, cur, inpSA);
|
|
cb(cur, "attn_residual", il);
|
|
|
|
// Save the tensor before post-attention norm for residual connection
|
|
ggml_tensor * ffn_residual = cur;
|
|
|
|
// Post-attention norm
|
|
ggml_tensor * attn_post_norm = build_norm(cur, model.layers[il].attn_post_norm, nullptr, LLM_NORM_RMS, il);
|
|
cb(attn_post_norm, "attn_post_norm", il);
|
|
|
|
// FFN layer (MoE or dense) - without residual connection
|
|
cur = build_layer_ffn(attn_post_norm, il);
|
|
cb(cur, "ffn_out", il);
|
|
|
|
// Residual connection for FFN - add to the tensor from before post_attention_layernorm
|
|
cur = ggml_add(ctx0, cur, ffn_residual);
|
|
cb(cur, "post_moe", il);
|
|
|
|
cur = build_cvec(cur, il);
|
|
cb(cur, "l_out", il);
|
|
|
|
// Input for next layer
|
|
inpL = cur;
|
|
}
|
|
cur = inpL;
|
|
|
|
// post-norm hidden state is input to both the LM head and the MTP head
|
|
cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1);
|
|
|
|
cb(cur, "h_nextn", -1);
|
|
res->t_h_nextn = cur;
|
|
|
|
if (crop_after_nextn(inp_out_ids)) {
|
|
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
|
|
}
|
|
|
|
cb(cur, "result_norm", -1);
|
|
res->t_embd = cur;
|
|
|
|
// LM head
|
|
cur = build_lora_mm(model.output, cur, model.output_s);
|
|
|
|
cb(cur, "result_output", -1);
|
|
res->t_logits = cur;
|
|
}
|
|
|
|
ggml_tensor * llama_model_qwen3next::graph::build_norm_gated(
|
|
ggml_tensor * input,
|
|
ggml_tensor * weights,
|
|
ggml_tensor * gate,
|
|
int layer) {
|
|
ggml_tensor * normalized = build_norm(input, weights, nullptr, LLM_NORM_RMS, layer);
|
|
ggml_tensor * gated_silu = ggml_silu(ctx0, gate);
|
|
|
|
return ggml_mul(ctx0, normalized, gated_silu);
|
|
}
|
|
|
|
ggml_tensor * llama_model_qwen3next::graph::build_layer_attn(
|
|
llm_graph_input_attn_kv * inp,
|
|
ggml_tensor * cur,
|
|
ggml_tensor * inp_pos,
|
|
int il) {
|
|
const int64_t n_embd_head = hparams.n_embd_head_v();
|
|
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
|
|
|
|
// Order: joint QG projection, QG split, Q norm, KV projection, K norm, RoPE, attention
|
|
|
|
// Qwen3Next uses a single Q projection that outputs query + gate
|
|
auto [Qcur_full, Kcur, Vcur] = build_qkv(model.layers[il], cur,
|
|
n_embd_head * 2, n_head,
|
|
n_embd_head, n_head_kv,
|
|
n_embd_head, n_head_kv,
|
|
il, false);
|
|
cb(Qcur_full, "Qcur_full", il);
|
|
cb(Kcur, "Kcur", il);
|
|
cb(Vcur, "Vcur", il);
|
|
|
|
Qcur_full = ggml_reshape_4d(ctx0, Qcur_full, n_embd_head * 2, n_head, n_tokens, 1);
|
|
|
|
// Split Q projection into query and gate
|
|
// The split should be along dimension 0 (the feature dimension)
|
|
ggml_tensor * Qcur = ggml_view_4d(ctx0, Qcur_full, n_embd_head, n_head, n_tokens, 1,
|
|
Qcur_full->nb[1], Qcur_full->nb[2], Qcur_full->nb[3], 0);
|
|
cb(Qcur, "Qcur_view", il);
|
|
|
|
ggml_tensor * gate =
|
|
ggml_view_4d(ctx0, Qcur_full, n_embd_head, n_head, n_tokens, 1,
|
|
Qcur_full->nb[1], Qcur_full->nb[2], Qcur_full->nb[3], n_embd_head * ggml_element_size(Qcur_full));
|
|
cb(gate, "gate", il);
|
|
|
|
Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
|
|
Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
|
|
|
|
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il);
|
|
cb(Qcur, "Qcur_normed", il);
|
|
|
|
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il);
|
|
cb(Kcur, "Kcur_normed", il);
|
|
|
|
Qcur = ggml_rope_ext(
|
|
ctx0, Qcur, inp_pos, nullptr,
|
|
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
ext_factor, attn_factor, beta_fast, beta_slow);
|
|
|
|
Kcur = ggml_rope_ext(
|
|
ctx0, Kcur, inp_pos, nullptr,
|
|
n_rot, rope_type, n_ctx_orig, freq_base,
|
|
freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
|
|
|
|
cb(Qcur, "Qcur", il);
|
|
cb(Kcur, "Kcur", il);
|
|
cb(Vcur, "Vcur", il);
|
|
|
|
const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
|
|
|
|
cur = build_attn(inp,
|
|
nullptr, nullptr, nullptr,
|
|
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
|
|
cb(cur, "attn_pregate", il);
|
|
|
|
gate = ggml_cont_2d(ctx0, gate, n_embd_head * n_head, n_tokens);
|
|
|
|
gate = ggml_sigmoid(ctx0, gate);
|
|
cb(gate, "gate_sigmoid", il);
|
|
|
|
cur = ggml_mul(ctx0, cur, gate);
|
|
cb(cur, "attn_gated", il);
|
|
|
|
cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s);
|
|
cb(cur, "attn_output", il);
|
|
|
|
return cur;
|
|
}
|
|
|
|
std::pair<ggml_tensor *, ggml_tensor *> llama_model_qwen3next::graph::build_qkvz(
|
|
ggml_tensor * input,
|
|
int il) {
|
|
const int64_t d_inner = hparams.ssm_d_inner;
|
|
const int64_t n_seqs = ubatch.n_seqs;
|
|
const int64_t head_k_dim = hparams.ssm_d_state;
|
|
const int64_t num_k_heads = hparams.ssm_n_group;
|
|
const int64_t num_v_heads = hparams.ssm_dt_rank;
|
|
const int64_t head_v_dim = d_inner / num_v_heads;
|
|
const int64_t n_seq_tokens = ubatch.n_seq_tokens;
|
|
|
|
if (model.layers[il].wqkv) {
|
|
// optimized path
|
|
ggml_tensor * qkv_mixed = build_lora_mm(model.layers[il].wqkv, input, model.layers[il].wqkv_s);
|
|
qkv_mixed = ggml_reshape_3d(ctx0, qkv_mixed, qkv_mixed->ne[0], n_seq_tokens, n_seqs);
|
|
cb(qkv_mixed, "linear_attn_qkv_mixed", il);
|
|
|
|
ggml_tensor * z = build_lora_mm(model.layers[il].wqkv_gate, input, model.layers[il].wqkv_gate_s);
|
|
cb(z, "z", il);
|
|
|
|
return { qkv_mixed, z };
|
|
} else {
|
|
// legacy (slower) path
|
|
ggml_tensor * mixed_qkvz = build_lora_mm(model.layers[il].ssm_in, input);
|
|
cb(mixed_qkvz, "linear_attn_mixed_qkvz", il);
|
|
|
|
int64_t qkvz_new_dim = 2 * head_k_dim + 2 * head_v_dim * (num_v_heads / num_k_heads);
|
|
ggml_tensor * mixed_qkvz_reshaped = ggml_reshape_4d(ctx0, mixed_qkvz, qkvz_new_dim, num_k_heads, n_seq_tokens, n_seqs);
|
|
|
|
// Split mixed_qkvz into query, key, value, z
|
|
int64_t split_sizes_qkvz[4] = {
|
|
head_k_dim, // query size
|
|
head_k_dim, // key size
|
|
head_v_dim * num_v_heads / num_k_heads, // value size
|
|
head_v_dim * num_v_heads / num_k_heads // z size
|
|
};
|
|
|
|
ggml_tensor * query =
|
|
ggml_view_4d(ctx0, mixed_qkvz_reshaped, split_sizes_qkvz[0], num_k_heads, n_seq_tokens, n_seqs,
|
|
mixed_qkvz_reshaped->nb[1], mixed_qkvz_reshaped->nb[2], mixed_qkvz_reshaped->nb[3], 0);
|
|
cb(query, "q", il);
|
|
|
|
ggml_tensor * key = ggml_view_4d(ctx0, mixed_qkvz_reshaped, split_sizes_qkvz[1], num_k_heads, n_seq_tokens, n_seqs,
|
|
mixed_qkvz_reshaped->nb[1], mixed_qkvz_reshaped->nb[2], mixed_qkvz_reshaped->nb[3],
|
|
split_sizes_qkvz[0] * ggml_element_size(mixed_qkvz_reshaped));
|
|
cb(key, "k", il);
|
|
|
|
ggml_tensor * value =
|
|
ggml_view_4d(ctx0, mixed_qkvz_reshaped, split_sizes_qkvz[2], num_k_heads, n_seq_tokens, n_seqs,
|
|
mixed_qkvz_reshaped->nb[1], mixed_qkvz_reshaped->nb[2], mixed_qkvz_reshaped->nb[3],
|
|
(split_sizes_qkvz[0] + split_sizes_qkvz[1]) * ggml_element_size(mixed_qkvz_reshaped));
|
|
cb(value, "v", il);
|
|
|
|
ggml_tensor * z = ggml_view_4d(ctx0, mixed_qkvz_reshaped, split_sizes_qkvz[3], num_k_heads, n_seq_tokens, n_seqs,
|
|
mixed_qkvz_reshaped->nb[1], mixed_qkvz_reshaped->nb[2], mixed_qkvz_reshaped->nb[3],
|
|
(split_sizes_qkvz[0] + split_sizes_qkvz[1] + split_sizes_qkvz[2]) * ggml_element_size(mixed_qkvz_reshaped));
|
|
z = ggml_cont(ctx0, z);
|
|
cb(z, "z", il);
|
|
|
|
// After creating query, key, and value_reshaped, reshape each to flatten the head dimensions
|
|
// query: [head_k_dim, num_k_heads, n_tokens, n_seqs] -> [head_k_dim * num_k_heads, n_tokens, n_seqs]
|
|
ggml_tensor * query_flat = ggml_cont_3d(ctx0, query, head_k_dim * num_k_heads, n_seq_tokens, n_seqs);
|
|
cb(query_flat, "query_flat", il);
|
|
|
|
// key: [head_k_dim, num_k_heads, n_tokens, n_seqs] -> [head_k_dim * num_k_heads, n_tokens, n_seqs]
|
|
ggml_tensor * key_flat = ggml_cont_3d(ctx0, key, head_k_dim * num_k_heads, n_seq_tokens, n_seqs);
|
|
cb(key_flat, "key_flat", il);
|
|
|
|
// value_reshaped: [head_v_dim, num_v_heads, n_tokens, n_seqs] -> [head_v_dim * num_v_heads, n_tokens, n_seqs]
|
|
ggml_tensor * value_flat = ggml_cont_3d(ctx0, value, head_v_dim * num_v_heads, n_seq_tokens, n_seqs);
|
|
cb(value_flat, "value_flat", il);
|
|
|
|
// Now concatenate along the feature dimension (dim 0) to get [conv_dim, n_tokens, n_seqs]
|
|
ggml_tensor * qkv_mixed = ggml_concat(ctx0, query_flat, key_flat, 0);
|
|
qkv_mixed = ggml_concat(ctx0, qkv_mixed, value_flat, 0);
|
|
cb(qkv_mixed, "qkv_mixed", il);
|
|
|
|
return { qkv_mixed, z };
|
|
}
|
|
}
|
|
|
|
ggml_tensor * llama_model_qwen3next::graph::build_layer_attn_linear(
|
|
llm_graph_input_rs * inp,
|
|
ggml_tensor * cur,
|
|
int il) {
|
|
const auto * mctx_cur = inp->mctx;
|
|
|
|
const int64_t d_inner = hparams.ssm_d_inner;
|
|
const int64_t n_seqs = ubatch.n_seqs;
|
|
const int64_t head_k_dim = hparams.ssm_d_state;
|
|
const int64_t num_k_heads = hparams.ssm_n_group;
|
|
const int64_t num_v_heads = hparams.ssm_dt_rank;
|
|
const int64_t head_v_dim = d_inner / num_v_heads;
|
|
const int64_t n_seq_tokens = ubatch.n_seq_tokens;
|
|
|
|
GGML_ASSERT(n_seqs != 0);
|
|
GGML_ASSERT(ubatch.equal_seqs());
|
|
GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);
|
|
|
|
// Input projections
|
|
auto qkvz = build_qkvz(cur, il);
|
|
ggml_tensor * qkv_mixed = qkvz.first;
|
|
ggml_tensor * z = qkvz.second;
|
|
|
|
ggml_tensor * mixed_ba = build_lora_mm(model.layers[il].ssm_beta_alpha, cur);
|
|
cb(mixed_ba, "linear_attn_mixed_ba", il);
|
|
|
|
// Reshape mixed_ba: [batch, seq_len, hidden_size] -> [batch, seq_len, num_k_heads, 2*num_v_heads/num_k_heads]
|
|
int64_t ba_new_dim = 2 * num_v_heads / num_k_heads;
|
|
ggml_tensor * mixed_ba_reshaped = ggml_reshape_4d(ctx0, mixed_ba, ba_new_dim, num_k_heads, n_seq_tokens, n_seqs);
|
|
|
|
// Split mixed_ba into b and a (beta and alpha parameters)
|
|
int64_t split_sizes_ba[2] = {
|
|
num_v_heads / num_k_heads, // beta size
|
|
num_v_heads / num_k_heads // alpha size
|
|
};
|
|
|
|
ggml_tensor * b = ggml_view_4d(ctx0, mixed_ba_reshaped, split_sizes_ba[0], num_k_heads, n_seq_tokens, n_seqs,
|
|
mixed_ba_reshaped->nb[1], mixed_ba_reshaped->nb[2], mixed_ba_reshaped->nb[3], 0);
|
|
cb(b, "b", il);
|
|
|
|
ggml_tensor * a = ggml_view_4d(ctx0, mixed_ba_reshaped, split_sizes_ba[1], num_k_heads, n_seq_tokens, n_seqs,
|
|
mixed_ba_reshaped->nb[1], mixed_ba_reshaped->nb[2], mixed_ba_reshaped->nb[3],
|
|
split_sizes_ba[0] * ggml_element_size(mixed_ba_reshaped));
|
|
cb(a, "a", il);
|
|
|
|
ggml_tensor * beta = ggml_sigmoid(ctx0, b);
|
|
|
|
// Reshape a to merge head dimensions: [batch, seq_len, num_k_heads, num_v_heads/num_k_heads] -> [batch, seq_len, num_v_heads]
|
|
ggml_tensor * alpha = ggml_cont_3d(ctx0, a, num_v_heads, n_seq_tokens, n_seqs);
|
|
|
|
ggml_tensor * alpha_biased = ggml_add(ctx0, alpha, model.layers[il].ssm_dt);
|
|
ggml_tensor * alpha_softplus = ggml_softplus(ctx0, alpha_biased);
|
|
cb(alpha_softplus, "a_softplus", il);
|
|
|
|
ggml_tensor * gate = ggml_mul(ctx0, alpha_softplus, model.layers[il].ssm_a); // -A_log.exp() * softplus
|
|
cb(gate, "gate", il);
|
|
|
|
beta = ggml_reshape_4d(ctx0, beta, 1, num_v_heads, n_seq_tokens, n_seqs);
|
|
gate = ggml_reshape_4d(ctx0, gate, 1, num_v_heads, n_seq_tokens, n_seqs);
|
|
|
|
ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);
|
|
ggml_tensor * ssm_states_all = mctx_cur->get_s_l(il);
|
|
|
|
ggml_tensor * conv_kernel = model.layers[il].ssm_conv1d;
|
|
const int64_t conv_kernel_size = conv_kernel->ne[0];
|
|
const int64_t conv_channels = d_inner + 2 * hparams.ssm_n_group * hparams.ssm_d_state;
|
|
|
|
ggml_tensor * conv_input = build_conv_state(inp, conv_states_all, qkv_mixed, conv_kernel_size, conv_channels, il);
|
|
|
|
ggml_tensor * state = build_rs(inp, ssm_states_all, hparams.n_embd_s(), n_seqs);
|
|
state = ggml_reshape_4d(ctx0, state, head_v_dim, head_v_dim, num_v_heads, n_seqs);
|
|
cb(state, "state_predelta", il);
|
|
|
|
ggml_tensor * conv_output_proper = ggml_ssm_conv(ctx0, conv_input, conv_kernel);
|
|
cb(conv_output_proper, "conv_output_raw", il);
|
|
|
|
ggml_tensor * conv_output_silu = ggml_silu(ctx0, conv_output_proper);
|
|
cb(conv_output_silu, "conv_output_silu", il);
|
|
|
|
ggml_tensor * conv_qkv_mix = conv_output_silu;
|
|
|
|
// Calculate the total conv dimension
|
|
int64_t qkv_dim = head_k_dim * num_k_heads * 2 + head_v_dim * num_v_heads;
|
|
int64_t nb1_qkv = ggml_row_size(conv_qkv_mix->type, qkv_dim);
|
|
|
|
// Extract the convolved Q, K, V from conv_output
|
|
ggml_tensor * q_conv = ggml_view_4d(ctx0, conv_qkv_mix, head_k_dim, num_k_heads, n_seq_tokens, n_seqs,
|
|
ggml_row_size(conv_qkv_mix->type, head_k_dim),
|
|
nb1_qkv,
|
|
nb1_qkv * n_seq_tokens,
|
|
0);
|
|
|
|
ggml_tensor * k_conv = ggml_view_4d(ctx0, conv_qkv_mix, head_k_dim, num_k_heads, n_seq_tokens, n_seqs,
|
|
ggml_row_size(conv_qkv_mix->type, head_k_dim),
|
|
nb1_qkv,
|
|
nb1_qkv * n_seq_tokens,
|
|
head_k_dim * num_k_heads * ggml_element_size(conv_qkv_mix));
|
|
|
|
ggml_tensor * v_conv = ggml_view_4d(ctx0, conv_qkv_mix, head_v_dim, num_v_heads, n_seq_tokens, n_seqs,
|
|
ggml_row_size(conv_qkv_mix->type, head_v_dim),
|
|
nb1_qkv,
|
|
nb1_qkv * n_seq_tokens,
|
|
ggml_row_size(conv_qkv_mix->type, 2 * head_k_dim * num_k_heads));
|
|
|
|
cb(q_conv, "q_conv", il);
|
|
cb(k_conv, "k_conv", il);
|
|
cb(v_conv, "v_conv", il);
|
|
|
|
|
|
const float eps_norm = hparams.f_norm_rms_eps;
|
|
|
|
q_conv = build_gdn_l2_norm(ctx0, q_conv, eps_norm);
|
|
k_conv = build_gdn_l2_norm(ctx0, k_conv, eps_norm);
|
|
|
|
//q_conv = ggml_cont_4d(ctx0, q_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs);
|
|
//k_conv = ggml_cont_4d(ctx0, k_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs);
|
|
//v_conv = ggml_cont_4d(ctx0, v_conv, head_v_dim, num_v_heads, n_seq_tokens, n_seqs);
|
|
|
|
// if head keys and value keys are different, repeat to force tensors into matching shapes
|
|
// TODO: avoid repeats for fused GDN, needs broadcast configuration for GDN op [TAG_GGML_GDN_BCAST]
|
|
if (num_k_heads != num_v_heads) {
|
|
GGML_ASSERT(num_v_heads % num_k_heads == 0);
|
|
int64_t repeat_factor = num_v_heads / num_k_heads;
|
|
|
|
// repeat interleave: reshape to (repeat part, 1, remaining part...), do repeat, then reshape back
|
|
ggml_tensor * q_reshaped = ggml_reshape_4d(ctx0, q_conv, head_k_dim, 1, num_k_heads, n_seq_tokens * n_seqs);
|
|
ggml_tensor * k_reshaped = ggml_reshape_4d(ctx0, k_conv, head_k_dim, 1, num_k_heads, n_seq_tokens * n_seqs);
|
|
|
|
// Repeat along the third dimension (the new dimension with size 1)
|
|
ggml_tensor * q_repeated =
|
|
ggml_repeat_4d(ctx0, q_reshaped, head_k_dim, repeat_factor, num_k_heads, n_seq_tokens * n_seqs);
|
|
ggml_tensor * k_repeated =
|
|
ggml_repeat_4d(ctx0, k_reshaped, head_k_dim, repeat_factor, num_k_heads, n_seq_tokens * n_seqs);
|
|
|
|
// Reshape back to merge the head and repeat dimensions
|
|
// From [head_dim, repeat_factor, num_k_heads, n_seq_tokens * n_seqs]
|
|
// Back to [head_dim, repeat_factor * num_k_heads, n_seq_tokens, n_seqs]
|
|
q_conv = ggml_reshape_4d(ctx0, q_repeated, head_k_dim, num_k_heads * repeat_factor, n_seq_tokens, n_seqs);
|
|
k_conv = ggml_reshape_4d(ctx0, k_repeated, head_k_dim, num_k_heads * repeat_factor, n_seq_tokens, n_seqs);
|
|
}
|
|
|
|
cb(q_conv, "q_conv_predelta", il);
|
|
cb(k_conv, "k_conv_predelta", il);
|
|
cb(v_conv, "v_conv_predelta", il);
|
|
|
|
ggml_tensor * output = build_recurrent_attn(inp, ssm_states_all, q_conv, k_conv, v_conv, gate, beta, state, il);
|
|
|
|
// z: [head_dim, n_heads, n_tokens, n_seqs] -> [n_heads * n_tokens * n_seqs, head_dim]
|
|
ggml_tensor * z_2d = ggml_reshape_4d(ctx0, z, head_v_dim, num_v_heads, n_seq_tokens, n_seqs);
|
|
|
|
// Apply gated normalization: self.norm(core_attn_out, z)
|
|
ggml_tensor * attn_out_norm = build_norm_gated(output, model.layers[il].ssm_norm, z_2d, il);
|
|
|
|
// Final reshape: [head_dim, n_heads, n_tokens, n_seqs] -> [n_heads * head_dim , n_tokens * n_seqs ]
|
|
ggml_tensor * final_output = ggml_reshape_2d(ctx0, attn_out_norm, head_v_dim * num_v_heads, n_seq_tokens * n_seqs);
|
|
cb(final_output, "final_output", il);
|
|
|
|
// Output projection
|
|
cur = build_lora_mm(model.layers[il].ssm_out, final_output);
|
|
cb(cur, "linear_attn_out", il);
|
|
|
|
return cur;
|
|
}
|
|
|
|
ggml_tensor * llama_model_qwen3next::graph::build_layer_ffn(ggml_tensor * cur, const int il) {
|
|
// Check if this is an MoE layer
|
|
if (model.layers[il].ffn_gate_inp != nullptr) {
|
|
// MoE branch
|
|
ggml_tensor * moe_out =
|
|
build_moe_ffn(cur,
|
|
model.layers[il].ffn_gate_inp,
|
|
model.layers[il].ffn_up_exps,
|
|
model.layers[il].ffn_gate_exps,
|
|
model.layers[il].ffn_down_exps,
|
|
nullptr,
|
|
n_expert, n_expert_used,
|
|
LLM_FFN_SILU, true,
|
|
hparams.expert_weights_scale,
|
|
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
|
|
nullptr, model.layers[il].ffn_gate_up_exps,
|
|
model.layers[il].ffn_up_exps_s,
|
|
model.layers[il].ffn_gate_exps_s,
|
|
model.layers[il].ffn_down_exps_s);
|
|
cb(moe_out, "ffn_moe_out", il);
|
|
|
|
// Add shared experts if present - following Qwen3Next reference implementation
|
|
if (model.layers[il].ffn_up_shexp != nullptr) {
|
|
ggml_tensor * ffn_shexp =
|
|
build_ffn(cur,
|
|
model.layers[il].ffn_up_shexp, NULL, model.layers[il].ffn_up_shexp_s,
|
|
model.layers[il].ffn_gate_shexp, NULL, model.layers[il].ffn_gate_shexp_s,
|
|
model.layers[il].ffn_down_shexp, NULL, model.layers[il].ffn_down_shexp_s,
|
|
NULL,
|
|
LLM_FFN_SILU, LLM_FFN_PAR, il);
|
|
cb(ffn_shexp, "ffn_shexp", il);
|
|
|
|
// Apply shared expert gating as in the reference implementation
|
|
// The shared expert has its own gate that is sigmoided
|
|
// Note: ffn_gate_inp_shexp is the shared expert gate (outputs 1 value per token)
|
|
ggml_tensor * shared_gate = build_lora_mm(model.layers[il].ffn_gate_inp_shexp, cur);
|
|
cb(shared_gate, "shared_expert_gate", il);
|
|
|
|
shared_gate = ggml_sigmoid(ctx0, shared_gate);
|
|
cb(shared_gate, "shared_expert_gate_sigmoid", il);
|
|
|
|
ffn_shexp = ggml_mul(ctx0, ffn_shexp, shared_gate);
|
|
cb(ffn_shexp, "ffn_shexp_gated", il);
|
|
|
|
cur = ggml_add(ctx0, moe_out, ffn_shexp);
|
|
cb(cur, "ffn_out", il);
|
|
} else {
|
|
cur = moe_out;
|
|
}
|
|
} else {
|
|
// Dense FFN branch (not currently used I believe)
|
|
cur = build_ffn(cur,
|
|
model.layers[il].ffn_up, NULL, NULL,
|
|
model.layers[il].ffn_gate, NULL, NULL,
|
|
model.layers[il].ffn_down, NULL, NULL,
|
|
NULL,
|
|
LLM_FFN_SILU, LLM_FFN_PAR, il);
|
|
cb(cur, "ffn_out", il);
|
|
}
|
|
return cur;
|
|
}
|
|
|
|
// LLM_GRAPH_TYPE_DECODER_MTP draft head for Qwen3-Next
|
|
llama_model_qwen3next::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params)
|
|
: llm_graph_context(params) {
|
|
GGML_ASSERT(hparams.n_layer_nextn > 0 && "QWEN3NEXT MTP requires n_layer_nextn > 0");
|
|
GGML_ASSERT(hparams.n_layer_nextn == 1 && "QWEN3NEXT MTP currently only supports a single MTP block");
|
|
|
|
const int64_t n_embd_head = hparams.n_embd_head_v();
|
|
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
|
|
|
|
const int il = hparams.n_layer();
|
|
const auto & layer = model.layers[il];
|
|
|
|
GGML_ASSERT(layer.nextn.eh_proj && "MTP block missing nextn.eh_proj");
|
|
GGML_ASSERT(layer.nextn.enorm && "MTP block missing nextn.enorm");
|
|
GGML_ASSERT(layer.nextn.hnorm && "MTP block missing nextn.hnorm");
|
|
GGML_ASSERT(layer.ffn_gate_inp && "MTP block missing ffn_gate_inp");
|
|
|
|
// TODO: extract in a common llm_graph_context::build_inp_embd_h()
|
|
auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), hparams.n_embd);
|
|
|
|
inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
|
|
ggml_set_input(inp->tokens);
|
|
|
|
inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens);
|
|
ggml_set_input(inp->embd);
|
|
|
|
// TODO: make static using `ggml_build_forward_select()`
|
|
// see llm_graph_context::build_inp_embd() for reference
|
|
ggml_tensor * tok_embd;
|
|
ASSERT_EMBD_OR_TOKEN(ubatch);
|
|
if (ubatch.token) {
|
|
ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd;
|
|
|
|
tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens);
|
|
} else {
|
|
tok_embd = inp->embd;
|
|
}
|
|
cb(tok_embd, "mtp_tok_embd", il);
|
|
|
|
inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens);
|
|
ggml_set_input(inp->h);
|
|
ggml_set_name(inp->h, "mtp_h_input");
|
|
|
|
ggml_tensor * h_embd = inp->h;
|
|
|
|
res->add_input(std::move(inp));
|
|
|
|
ggml_tensor * inp_pos = build_inp_pos();
|
|
ggml_tensor * inp_out_ids = build_inp_out_ids();
|
|
|
|
auto * inp_attn = build_attn_inp_kv();
|
|
|
|
ggml_tensor * h_norm = build_norm(h_embd, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il);
|
|
cb(h_norm, "mtp_hnorm", il);
|
|
|
|
ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il);
|
|
cb(e_norm, "mtp_enorm", il);
|
|
|
|
ggml_tensor * concat = ggml_concat(ctx0, e_norm, h_norm, /*dim=*/ 0);
|
|
cb(concat, "mtp_concat", il);
|
|
|
|
ggml_tensor * cur = build_lora_mm(layer.nextn.eh_proj, concat, layer.nextn.eh_proj_s);
|
|
cb(cur, "mtp_eh_proj", il);
|
|
|
|
ggml_tensor * inpSA = cur;
|
|
|
|
cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
|
|
cb(cur, "mtp_attn_norm", il);
|
|
|
|
auto [Qcur_full, Kcur, Vcur] = build_qkv(layer, cur,
|
|
n_embd_head * 2, n_head,
|
|
n_embd_head, n_head_kv,
|
|
n_embd_head, n_head_kv,
|
|
il, false);
|
|
cb(Qcur_full, "mtp_Qcur_full", il);
|
|
|
|
ggml_tensor * Qcur = ggml_view_3d(ctx0, Qcur_full,
|
|
n_embd_head, n_head, n_tokens,
|
|
ggml_element_size(Qcur_full) * n_embd_head * 2,
|
|
ggml_element_size(Qcur_full) * n_embd_head * 2 * n_head,
|
|
0);
|
|
Qcur = build_norm(Qcur, layer.attn_q_norm, nullptr, LLM_NORM_RMS, il);
|
|
cb(Qcur, "mtp_Qcur_normed", il);
|
|
|
|
Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
|
|
Kcur = build_norm(Kcur, layer.attn_k_norm, nullptr, LLM_NORM_RMS, il);
|
|
cb(Kcur, "mtp_Kcur_normed", il);
|
|
|
|
Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
|
|
|
|
Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr,
|
|
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
ext_factor, attn_factor, beta_fast, beta_slow);
|
|
Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr,
|
|
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
ext_factor, attn_factor, beta_fast, beta_slow);
|
|
|
|
cb(Qcur, "mtp_Qcur", il);
|
|
cb(Kcur, "mtp_Kcur", il);
|
|
cb(Vcur, "mtp_Vcur", il);
|
|
|
|
const float kq_scale = hparams.f_attention_scale == 0.0f
|
|
? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
|
|
|
|
cur = build_attn(inp_attn,
|
|
nullptr, nullptr, nullptr,
|
|
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
|
|
cb(cur, "mtp_attn_pregate", il);
|
|
|
|
ggml_tensor * gate = ggml_view_3d(ctx0, Qcur_full,
|
|
n_embd_head, n_head, n_tokens,
|
|
ggml_element_size(Qcur_full) * n_embd_head * 2,
|
|
ggml_element_size(Qcur_full) * n_embd_head * 2 * n_head,
|
|
ggml_element_size(Qcur_full) * n_embd_head);
|
|
|
|
gate = ggml_cont_2d(ctx0, gate, n_embd_head * n_head, n_tokens);
|
|
cb(gate, "mtp_gate", il);
|
|
|
|
cur = ggml_mul(ctx0, cur, ggml_sigmoid(ctx0, gate));
|
|
cur = build_lora_mm(layer.wo, cur, layer.wo_s);
|
|
cb(cur, "mtp_attn_out", il);
|
|
|
|
if (inp_out_ids) {
|
|
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
|
|
inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
|
|
}
|
|
|
|
cur = ggml_add(ctx0, cur, inpSA);
|
|
cb(cur, "mtp_attn_residual", il);
|
|
|
|
ggml_tensor * ffn_residual = cur;
|
|
cur = build_norm(cur, layer.attn_post_norm, nullptr, LLM_NORM_RMS, il);
|
|
cb(cur, "mtp_attn_post_norm", il);
|
|
|
|
// MoE FFN — routed experts plus gated shared expert (mirrors the trunk).
|
|
ggml_tensor * moe_out =
|
|
build_moe_ffn(cur,
|
|
layer.ffn_gate_inp,
|
|
layer.ffn_up_exps,
|
|
layer.ffn_gate_exps,
|
|
layer.ffn_down_exps,
|
|
nullptr,
|
|
n_expert, n_expert_used,
|
|
LLM_FFN_SILU, true,
|
|
hparams.expert_weights_scale,
|
|
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,
|
|
nullptr, layer.ffn_gate_up_exps,
|
|
layer.ffn_up_exps_s,
|
|
layer.ffn_gate_exps_s,
|
|
layer.ffn_down_exps_s);
|
|
cb(moe_out, "mtp_ffn_moe_out", il);
|
|
|
|
if (layer.ffn_up_shexp != nullptr) {
|
|
ggml_tensor * ffn_shexp =
|
|
build_ffn(cur,
|
|
layer.ffn_up_shexp, nullptr, layer.ffn_up_shexp_s,
|
|
layer.ffn_gate_shexp, nullptr, layer.ffn_gate_shexp_s,
|
|
layer.ffn_down_shexp, nullptr, layer.ffn_down_shexp_s,
|
|
nullptr,
|
|
LLM_FFN_SILU, LLM_FFN_PAR, il);
|
|
cb(ffn_shexp, "mtp_ffn_shexp", il);
|
|
|
|
ggml_tensor * shared_gate = build_lora_mm(layer.ffn_gate_inp_shexp, cur);
|
|
shared_gate = ggml_sigmoid(ctx0, shared_gate);
|
|
cb(shared_gate, "mtp_shared_expert_gate_sigmoid", il);
|
|
|
|
ffn_shexp = ggml_mul(ctx0, ffn_shexp, shared_gate);
|
|
cb(ffn_shexp, "mtp_ffn_shexp_gated", il);
|
|
|
|
cur = ggml_add(ctx0, moe_out, ffn_shexp);
|
|
} else {
|
|
cur = moe_out;
|
|
}
|
|
cb(cur, "mtp_ffn_out", il);
|
|
|
|
cur = ggml_add(ctx0, cur, ffn_residual);
|
|
cb(cur, "mtp_post_ffn", il);
|
|
|
|
ggml_tensor * head_norm_w = layer.nextn.shared_head_norm
|
|
? layer.nextn.shared_head_norm
|
|
: model.output_norm;
|
|
GGML_ASSERT(head_norm_w && "QWEN3NEXT MTP: missing both nextn.shared_head_norm and output_norm");
|
|
cur = build_norm(cur, head_norm_w, nullptr, LLM_NORM_RMS, -1);
|
|
|
|
cb(cur, "h_nextn", -1);
|
|
res->t_h_nextn = cur;
|
|
|
|
ggml_tensor * head_w = layer.nextn.shared_head_head ? layer.nextn.shared_head_head : model.output;
|
|
ggml_tensor * head_s = layer.nextn.shared_head_head ? layer.nextn.shared_head_head_s : model.output_s;
|
|
GGML_ASSERT(head_w && "QWEN3NEXT MTP: missing LM head (nextn.shared_head_head or model.output)");
|
|
cur = build_lora_mm(head_w, cur, head_s);
|
|
cb(cur, "result_output", -1);
|
|
|
|
res->t_logits = cur;
|
|
}
|