Files
llama.cpp/src/models/inkling2.cpp
T
Daniel HanandSigbjørn Skjæret a1f4222608 model : load ffn_gate_up_exps scales (#32657)
* model: load `ffn_gate_up_exps` scales

The generic scale pass loads per-expert scales for ffn_gate_exps,
ffn_up_exps and ffn_down_exps but not for the merged ffn_gate_up_exps.
Load ~ffn_gate_up_exps_s there~ and expose it as ffn_up_exps_s ~when the layer
has no separate up tensor,~ since build_moe_ffn applies up_exps_s to the
merged gate_up output. Inkling 2 now passes the expert scales to
build_moe_ffn like the other MoE models.

* Update src/llama-model.cpp

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>

* Update src/llama-model.h

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>

---------

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
2026-11-20 13:11:28 +01:00

413 lines
20 KiB
C++

#include "models.h"
#include "../llama-memory-hybrid-iswa.h"
void llama_model_inkling2::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
ml.get_key(LLM_KV_ATTENTION_QK_CLIP, hparams.f_qk_clip);
ml.get_key(LLM_KV_ATTN_RES_BLOCK_SIZE, hparams.attn_res_block_size);
ml.get_key(LLM_KV_SHORTCONV_L_CACHE, hparams.n_shortconv_l_cache);
// relative-position bias, and the log-scaled positions of the full-attention layers
ml.get_key(LLM_KV_ATTENTION_RELATIVE_LENGTH, hparams.n_relative);
ml.get_key(LLM_KV_ATTENTION_RELATIVE_EXTENT, hparams.n_relative_extent);
ml.get_key(LLM_KV_ATTENTION_RELATIVE_CLIP, hparams.f_relative_clip);
ml.get_key(LLM_KV_ATTENTION_LOG_SCALE_FLOOR, hparams.n_log_scale_floor);
ml.get_key(LLM_KV_ATTENTION_LOG_SCALE_ALPHA, hparams.f_log_scale_alpha);
// the embedding and output tensors are padded past the vocabulary
ml.get_key(LLM_KV_VOCAB_SIZE, hparams.n_vocab_padded);
ml.get_key(LLM_KV_VOCAB_SIZE_UNPADDED, hparams.n_vocab_unpadded);
// MoE
ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);
ml.get_key(LLM_KV_ACTIVATION_SITU_BETA, hparams.situ_beta);
ml.get_key(LLM_KV_ACTIVATION_SITU_LINEAR_BETA, hparams.situ_linear_beta);
ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);
hparams.f_final_logit_softcapping = 0.0f;
ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping, false);
// sliding window
ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);
uint32_t n_pattern = 0;
ml.get_arr_n(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, n_pattern);
if (n_pattern != hparams.n_layer() || !hparams.n_relative || !hparams.n_relative_extent || !hparams.n_swa ||
!hparams.attn_res_block_size || !hparams.n_log_scale_floor || hparams.n_shortconv_l_cache < 2 ||
hparams.n_shortconv_l_cache > 64 || hparams.n_relative_extent > INT32_MAX || hparams.n_swa > INT32_MAX ||
!hparams.n_vocab_unpadded || hparams.n_vocab_unpadded > hparams.n_vocab_padded) {
throw std::runtime_error("inkling2: invalid dimensions, vocabulary, or layer pattern");
}
for (float value : {hparams.f_norm_rms_eps, hparams.f_qk_clip, hparams.f_relative_clip, hparams.situ_beta, hparams.situ_linear_beta}) {
if (!std::isfinite(value) || value <= 0.0f) {
throw std::runtime_error("inkling2: normalization, clipping and scaling parameters must be finite and positive");
}
}
if (!std::isfinite(hparams.f_log_scale_alpha) || hparams.f_log_scale_alpha < 0.0f) {
throw std::runtime_error("inkling2: invalid logarithmic scale");
}
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
hparams.expert_router_f32 = true;
hparams.expert_weights_norm_eps = 1e-20f;
// no RoPE, the relative-position bias carries the positions
hparams.rope_pattern.fill(false);
// the convolution state keeps K, V, the attention output and the FFN output of the widest layer
uint64_t n_channels_max = 0;
for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
if (!hparams.n_head_kv(il) || hparams.n_head(il) % hparams.n_head_kv(il) ||
hparams.n_embd_head_k(il) != hparams.n_embd_head_v(il) || !hparams.n_expert_used(il) ||
hparams.n_expert_used(il) > hparams.n_expert || !hparams.n_ff_exp(il)) {
throw std::runtime_error("inkling2: invalid attention or expert dimensions");
}
n_channels_max = std::max<uint64_t>(n_channels_max, 2ull * hparams.n_embd_k_gqa(il) + 2ull * hparams.n_embd);
}
if (n_channels_max * (hparams.n_shortconv_l_cache - 1) > UINT32_MAX) {
throw std::runtime_error("inkling2: convolution state is too large");
}
hparams.n_sconv_channels = n_channels_max;
type = LLM_TYPE_UNKNOWN;
}
void llama_model_inkling2::load_arch_tensors(llama_model_loader &) {
LLAMA_LOAD_LOCALS;
if (hparams.n_vocab_unpadded != vocab.n_tokens()) {
throw std::runtime_error("inkling2: tokenizer size must equal the unpadded vocabulary size");
}
const int64_t n_vocab_padded = hparams.n_vocab_padded;
const int64_t n_rel = hparams.n_relative;
const int64_t n_conv = hparams.n_shortconv_l_cache;
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab_padded}, 0);
tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {n_embd}, 0);
// output
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab_padded}, 0);
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
output_res_query = create_tensor(tn(LLM_TENSOR_OUTPUT_RES_QUERY, "weight"), {n_embd}, 0);
for (int i = 0; i < n_layer; ++i) {
auto & layer = layers[i];
const int64_t n_head = hparams.n_head(i);
const int64_t n_head_kv = hparams.n_head_kv(i);
const int64_t n_embd_head = hparams.n_embd_head_k(i);
const int64_t n_ff_exp = hparams.n_ff_exp(i);
const int64_t n_extent = hparams.is_swa(i) ? hparams.n_swa : hparams.n_relative_extent;
// attention
layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head*n_head}, 0);
layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_head*n_head_kv}, 0);
layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_head*n_head_kv}, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head*n_head, n_embd}, 0);
layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_embd_head*n_head}, 0);
layer.attn_q_clip = create_tensor(tn(LLM_TENSOR_ATTN_Q_CLIP, "weight", i), {n_embd_head, n_head}, 0);
layer.attn_sink_proj = create_tensor(tn(LLM_TENSOR_ATTN_SINK_PROJ, "weight", i), {n_embd, n_head}, 0);
// relative-position bias
layer.attn_rel = create_tensor(tn(LLM_TENSOR_ATTN_REL, "weight", i), {n_embd, n_rel*n_head}, 0);
layer.attn_rel_bias = create_tensor(tn(LLM_TENSOR_ATTN_REL_BIAS, "weight", i), {n_rel, n_extent}, 0);
layer.attn_rel_clip = create_tensor(tn(LLM_TENSOR_ATTN_REL_CLIP, "weight", i), {n_rel, n_head}, 0);
// short convolutions
layer.attn_k_conv = create_tensor(tn(LLM_TENSOR_ATTN_K_CONV, "weight", i), {n_conv, n_embd_head*n_head_kv}, 0);
layer.attn_v_conv = create_tensor(tn(LLM_TENSOR_ATTN_V_CONV, "weight", i), {n_conv, n_embd_head*n_head_kv}, 0);
layer.attn_post_conv = create_tensor(tn(LLM_TENSOR_ATTN_POST_CONV, "weight", i), {n_conv, n_embd}, 0);
layer.ffn_post_conv = create_tensor(tn(LLM_TENSOR_FFN_POST_CONV, "weight", i), {n_conv, n_embd}, 0);
if (hparams.is_swa(i)) {
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
} else {
// full-attention layers: separate Q, K and V input norms, and input-dependent taps for the K and V convolutions
layer.attn_q_inp_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_INP_NORM, "weight", i), {n_embd}, 0);
layer.attn_k_inp_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_INP_NORM, "weight", i), {n_embd}, 0);
layer.attn_v_inp_norm = create_tensor(tn(LLM_TENSOR_ATTN_V_INP_NORM, "weight", i), {n_embd}, 0);
layer.attn_k_conv_proj = create_tensor(tn(LLM_TENSOR_ATTN_K_CONV_PROJ, "weight", i), {n_embd, n_conv*n_head_kv}, 0);
layer.attn_v_conv_proj = create_tensor(tn(LLM_TENSOR_ATTN_V_CONV_PROJ, "weight", i), {n_embd, n_conv*n_head_kv}, 0);
}
// block attention residuals
if (i > 0) {
layer.attn_res_query = create_tensor(tn(LLM_TENSOR_ATTN_RES_QUERY, "weight", i), {n_embd, hparams.is_swa(i) ? 1 : 3}, 0);
}
layer.ffn_res_query = create_tensor(tn(LLM_TENSOR_FFN_RES_QUERY, "weight", i), {n_embd}, 0);
// MoE
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED);
layer.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", i), {n_embd, 2*n_ff_exp, n_expert}, 0);
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
}
}
std::unique_ptr<llm_graph_context> llama_model_inkling2::build_arch_graph(const llm_graph_params & params) const {
return std::make_unique<graph>(params);
}
// scale each row down to an L2 norm of at most threshold, after the optional per-channel scale
ggml_tensor * llama_model_inkling2::graph::clip(ggml_tensor * cur, ggml_tensor * scale, float threshold) {
if (scale) {
cur = ggml_mul(ctx0, cur, scale);
}
ggml_tensor * norm = ggml_sqrt(ctx0, ggml_sum_rows(ctx0, ggml_sqr(ctx0, cur)));
norm = ggml_scale_bias(ctx0, norm, 1.0f, 1e-6f);
ggml_tensor * factor = ggml_div(ctx0, ggml_fill(ctx0, norm, threshold), norm);
return ggml_mul(ctx0, cur, ggml_clamp(ctx0, factor, 0.0f, 1.0f));
}
// attention over the depth: mix the earlier block outputs and cur with learned queries
ggml_tensor * llama_model_inkling2::graph::mix(const std::vector<ggml_tensor *> & blocks, ggml_tensor * cur, ggml_tensor * query) {
ggml_tensor * states = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens);
for (ggml_tensor * block : blocks) {
states = ggml_concat(ctx0, ggml_reshape_3d(ctx0, block, n_embd, 1, n_tokens), states, 1);
}
states = ggml_rms_norm(ctx0, states, 1e-6f);
ggml_tensor * queries = ggml_repeat(ctx0, query, ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_embd, query->ne[1], n_tokens));
ggml_tensor * scores = ggml_mul_mat(ctx0, states, queries);
scores = ggml_soft_max_ext(ctx0, scores, nullptr, 1.0f/sqrtf(n_embd), 0.0f);
return ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, states)), scores);
}
// causal short convolution with a residual; dynamic adds input-dependent taps per head
// the last n_conv - 1 inputs of each sequence live in the recurrent state at channel offset `offset`
ggml_tensor * llama_model_inkling2::graph::convolve(
ggml_tensor * cur,
ggml_tensor * weight,
ggml_tensor * dynamic,
ggml_tensor * state,
ggml_tensor * history,
llm_graph_input_rs * inp,
int64_t offset) {
const int64_t n_conv = hparams.n_shortconv_l_cache;
const int64_t n_channels = cur->ne[0];
const int64_t n_seq_tokens = ubatch.n_seq_tokens;
const int64_t n_seqs = ubatch.n_seqs;
ggml_tensor * past = ggml_view_3d(ctx0, history, n_conv - 1, n_channels, n_seqs, history->nb[1], history->nb[2], offset*history->nb[1]);
ggml_tensor * x = ggml_transpose(ctx0, ggml_reshape_3d(ctx0, cur, n_channels, n_seq_tokens, n_seqs));
x = ggml_concat(ctx0, past, x, 0);
build_conv_state_store(inp, state, x, n_conv - 1, offset*(n_conv - 1)*ggml_element_size(state));
ggml_tensor * out = ggml_ssm_conv(ctx0, x, weight);
out = ggml_reshape_2d(ctx0, out, n_channels, n_tokens);
if (dynamic) {
const int64_t n_heads = dynamic->ne[0]/n_conv;
dynamic = ggml_reshape_3d(ctx0, dynamic, n_conv, n_heads, n_tokens);
for (int64_t lag = 0; lag < n_conv; ++lag) {
ggml_tensor * tap = ggml_view_3d(ctx0, dynamic, 1, n_heads, n_tokens, dynamic->nb[1], dynamic->nb[2], lag*dynamic->nb[0]);
ggml_tensor * window = ggml_view_3d(ctx0, x, n_seq_tokens, n_channels, n_seqs, x->nb[1], x->nb[2], (n_conv - 1 - lag)*x->nb[0]);
window = ggml_cont(ctx0, ggml_transpose(ctx0, window));
window = ggml_reshape_3d(ctx0, window, n_channels/n_heads, n_heads, n_tokens);
ggml_tensor * term = ggml_reshape_2d(ctx0, ggml_mul(ctx0, window, tap), n_channels, n_tokens);
out = ggml_add(ctx0, out, term);
}
}
return ggml_scale(ctx0, ggml_add(ctx0, cur, out), 1.0f/sqrtf((dynamic ? 2 : 1)*n_conv + 1));
}
llama_model_inkling2::graph::graph(const llm_graph_params & params) : llm_graph_context(params) {
auto * inp = build_inp_mem_hybrid_iswa_rel(hparams.n_relative_extent, hparams.n_swa);
auto * attn = inp->get_attn();
auto * recr = inp->get_recr();
ggml_tensor * inp_pos = build_inp_pos();
ggml_tensor * inp_out_ids = build_inp_out_ids();
// log-scaled positions: 1 + alpha*log(max(1, (pos + 1)/floor)), multiplies q and r of the full-attention layers
ggml_tensor * tau = ggml_scale_bias(ctx0, ggml_cast(ctx0, inp_pos, GGML_TYPE_F32), 1.0f, 1.0f);
tau = ggml_log(ctx0, ggml_clamp(ctx0, ggml_scale(ctx0, tau, 1.0f/hparams.n_log_scale_floor), 1.0f, INFINITY));
tau = ggml_reshape_3d(ctx0, ggml_scale_bias(ctx0, tau, hparams.f_log_scale_alpha, 1.0f), 1, 1, n_tokens);
ggml_tensor * cur = build_inp_embd(model.tok_embd);
// important: do not normalize raw embeddings input (i.e. encoded image embeddings)
if (ubatch.token) {
cur = build_norm(cur, model.tok_norm, nullptr, LLM_NORM_RMS, -1);
}
std::vector<ggml_tensor *> blocks;
for (int il = 0; il < n_layer; ++il) {
const auto & layer = model.layers[il];
const bool is_swa = hparams.is_swa(il);
const int64_t n_head = hparams.n_head(il);
const int64_t n_head_kv = hparams.n_head_kv(il);
const int64_t n_embd_head = hparams.n_embd_head_k(il);
const int64_t n_embd_k_gqa = n_embd_head*n_head_kv;
ggml_tensor * state = recr->mctx->get_r_l(il);
ggml_tensor * history = build_rs(recr, state, hparams.n_embd_r(), ubatch.n_seqs);
history = ggml_reshape_3d(ctx0, history, hparams.n_shortconv_l_cache - 1, hparams.n_sconv_channels, ubatch.n_seqs);
ggml_tensor * inpSA = cur;
// attention inputs: SWA layers mix one, full-attention layers one each for Q, K and V
ggml_tensor * inputs[3] = {cur, cur, cur};
if (layer.attn_res_query) {
ggml_tensor * mixed = mix(blocks, cur, layer.attn_res_query);
for (int j = 0; j < 3; ++j) {
inputs[j] = ggml_cont(ctx0, ggml_view_2d(ctx0, mixed, n_embd, n_tokens, mixed->nb[2], (is_swa ? 0 : j)*mixed->nb[1]));
}
}
if (is_swa) {
inputs[0] = build_norm(inputs[0], layer.attn_norm, nullptr, LLM_NORM_RMS, il);
inputs[1] = inputs[2] = inputs[0];
} else {
inputs[0] = build_norm(inputs[0], layer.attn_q_inp_norm, nullptr, LLM_NORM_RMS, il);
inputs[1] = build_norm(inputs[1], layer.attn_k_inp_norm, nullptr, LLM_NORM_RMS, il);
inputs[2] = build_norm(inputs[2], layer.attn_v_inp_norm, nullptr, LLM_NORM_RMS, il);
}
// self-attention
{
ggml_tensor * Qcur = build_lora_mm(layer.wq, inputs[0]);
ggml_tensor * Kcur = convolve(build_lora_mm(layer.wk, inputs[1]), layer.attn_k_conv,
is_swa ? nullptr : build_lora_mm(layer.attn_k_conv_proj, inputs[1]), state, history, recr, 0);
ggml_tensor * Vcur = convolve(build_lora_mm(layer.wv, inputs[2]), layer.attn_v_conv,
is_swa ? nullptr : build_lora_mm(layer.attn_v_conv_proj, inputs[2]), state, history, recr, n_embd_k_gqa);
ggml_tensor * Rcur = build_lora_mm(layer.attn_rel, inputs[0]);
Qcur = clip(ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), layer.attn_q_clip, hparams.f_qk_clip);
Kcur = clip(ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), nullptr, hparams.f_qk_clip);
Vcur = ggml_rms_norm(ctx0, ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens), hparams.f_norm_rms_eps);
Rcur = clip(ggml_reshape_3d(ctx0, Rcur, hparams.n_relative, n_head, n_tokens), layer.attn_rel_clip, hparams.f_relative_clip);
if (!is_swa) {
Qcur = ggml_mul(ctx0, Qcur, tau);
Rcur = ggml_mul(ctx0, Rcur, tau);
}
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
cb(Vcur, "Vcur", il);
cb(Rcur, "Rcur", il);
ggml_tensor * rel_pos = is_swa ? inp->self_rel_pos_swa : inp->self_rel_pos;
const int64_t n_stream = (is_swa ? attn->self_attn_mask_swa.raw : attn->self_attn_mask.raw)->ne[3];
const int64_t n_kv = rel_pos->ne[0];
ggml_tensor * sinks = build_lora_mm(layer.attn_sink_proj, inputs[0]);
// project r on the bias table first, then gather by distance: only the gathered bias is n_kv x n_tokens x n_head
ggml_tensor * table = ggml_mul_mat(ctx0, ggml_pad(ctx0, layer.attn_rel_bias, 0, 1, 0, 0), Rcur);
// the sink logit t is per query: exp(s - t)/(1 + sum(exp(s - t))) = exp(s)/(exp(t) + sum(exp(s))), with a zero sink
table = ggml_sub(ctx0, table, ggml_reshape_3d(ctx0, sinks, 1, n_head, n_tokens));
table = ggml_cont(ctx0, ggml_permute(ctx0, table, 1, 0, 2, 3));
ggml_tensor * kq_b = ggml_get_rows(ctx0, table, rel_pos);
kq_b = ggml_reshape_4d(ctx0, kq_b, n_head, n_kv, n_tokens/n_stream, n_stream);
kq_b = ggml_cont(ctx0, ggml_permute(ctx0, kq_b, 2, 0, 1, 3));
cb(kq_b, "kq_b", il);
sinks = ggml_fill(ctx0, ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_head), 0.0f);
cur = build_attn(inp,
nullptr, nullptr, nullptr,
Qcur, Kcur, Vcur, kq_b, sinks, 1.0f/sqrtf(n_embd_head), il);
cur = ggml_mul(ctx0, cur, ggml_sigmoid(ctx0, build_lora_mm(layer.wqkv_gate, inputs[0])));
cur = convolve(build_lora_mm(layer.wo, cur), layer.attn_post_conv, nullptr, state, history, recr, 2*n_embd_k_gqa);
cb(cur, "attn_out", il);
}
// the first layer of each block starts a new residual stream, the earlier one joins the mixed blocks
if (il % hparams.attn_res_block_size == 0) {
blocks.push_back(inpSA);
} else {
cur = ggml_add(ctx0, inpSA, cur);
}
ggml_tensor * ffn_inp = cur;
cb(ffn_inp, "ffn_inp", il);
// MoE
cur = ggml_reshape_2d(ctx0, mix(blocks, cur, layer.ffn_res_query), n_embd, n_tokens);
cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
cur = build_moe_ffn(cur,
layer.ffn_gate_inp,
nullptr,
nullptr,
layer.ffn_down_exps,
layer.ffn_exp_probs_b,
hparams.n_expert,
hparams.n_expert_used(il),
LLM_FFN_SITU, true,
hparams.expert_weights_scale,
LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,
il,
nullptr,
layer.ffn_gate_up_exps,
layer.ffn_up_exps_s,
layer.ffn_gate_exps_s,
layer.ffn_down_exps_s);
cur = convolve(cur, layer.ffn_post_conv, nullptr, state, history, recr, 2*n_embd_k_gqa + n_embd);
cb(cur, "ffn_out", il);
cur = ggml_add(ctx0, ffn_inp, cur);
cb(cur, "l_out", il);
}
cur = ggml_reshape_2d(ctx0, mix(blocks, cur, model.output_res_query), n_embd, n_tokens);
cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1);
if (inp_out_ids) {
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
}
cb(cur, "result_norm", -1);
res->t_embd = cur;
// lm_head
cur = build_lora_mm(model.output, cur);
ggml_prec_set_acc(cur, GGML_PREC_F32);
if (hparams.f_final_logit_softcapping != 0.0f) {
const float cap = hparams.f_final_logit_softcapping;
cur = ggml_scale(ctx0, ggml_tanh(ctx0, ggml_scale(ctx0, cur, 1.0f/cap)), cap);
}
// drop the padded vocabulary
cur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, hparams.n_vocab_unpadded, cur->ne[1], cur->nb[1], 0));
cb(cur, "result_output", -1);
res->t_logits = cur;
}