Files
llama.cpp/src/models/zaya1.cpp
T
Xuan-Son Nguyen 19bc41dbb4 model: apply .scale tensors automatically + fix correctness (#32669)
* model: apply .scale tensors automatically

* rm build_moe_ffn "_s" overload

* use llama_prec_policy

* fix ci

* build_lora_mm_id: rm w_s
2026-11-21 16:30:01 +01:00

338 lines
17 KiB
C++

#include "models.h"
#include "llama-memory-recurrent.h"
void llama_model_zaya1::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);
ml.get_key(LLM_KV_ROUTER_HIDDEN_SIZE, hparams.n_router_hidden);
ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);
if (hparams.ssm_d_conv != 2) {
throw std::runtime_error("zaya1: only ssm.conv_kernel == 2 is supported");
}
// the two kv heads are v_cur and v_del, and the router picks one expert
for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
if (hparams.n_head_kv(il) != 2 || hparams.n_embd_head_v(il) != hparams.n_embd_head_k(il) || hparams.n_expert_used(il) != 1) {
throw std::runtime_error("zaya1: only 2 kv heads of equal k/v size and 1 used expert are supported");
}
}
hparams.n_cca_qk = (hparams.n_head() + hparams.n_head_kv()) * hparams.n_embd_head_k();
std::fill(hparams.is_recr_impl.begin(), hparams.is_recr_impl.end(), true);
switch (hparams.n_layer()) {
case 40: type = LLM_TYPE_8B_A1B; break;
default: type = LLM_TYPE_UNKNOWN;
}
}
void llama_model_zaya1::load_arch_tensors(llama_model_loader &) {
LLAMA_LOAD_LOCALS;
const int64_t n_embd_head = n_embd_head_k;
const int64_t n_group = n_head + n_head_kv;
const int64_t n_qk = hparams.n_cca_qk;
const int64_t d_conv = hparams.ssm_d_conv;
const int64_t n_ff_exp = hparams.n_ff_exp();
const int64_t n_router = hparams.n_router_hidden;
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
res_scale_in = create_tensor(tn(LLM_TENSOR_RES_SCALE_IN, "weight", 0), {n_embd}, 0);
res_scale_in_b = create_tensor(tn(LLM_TENSOR_RES_SCALE_IN, "bias", 0), {n_embd}, 0);
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
if (output == nullptr) {
output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
}
for (int i = 0; i < n_layer; ++i) {
auto & layer = layers[i];
auto & z = layer.zaya1;
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
// the two value heads are v_cur and v_del
create_tensor_qkv(layer, i, n_embd, n_embd_head * n_head, n_embd_head * n_head_kv, n_embd_v_gqa, 0);
layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "weight", i), {d_conv, n_qk}, 0);
layer.ssm_conv1d_b = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "bias", i), {n_qk}, 0);
z.cca_conv_grp = create_tensor(tn(LLM_TENSOR_CCA_CONV_GRP, "weight", i), {n_embd_head, n_embd_head, n_group, d_conv}, 0);
z.cca_conv_grp_b = create_tensor(tn(LLM_TENSOR_CCA_CONV_GRP, "bias", i), {n_qk}, 0);
z.cca_temp = create_tensor(tn(LLM_TENSOR_CCA_TEMP, "weight", i), {n_head_kv}, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head * n_head, n_embd}, 0);
z.attn_res_hs = create_tensor(tn(LLM_TENSOR_ATTN_RES_SCALE_HS, "weight", i), {n_embd}, 0);
layer.attn_resid_scale = create_tensor(tn(LLM_TENSOR_ATTN_RESID_SCALE, "weight", i), {n_embd}, 0);
layer.attn_resid_scale_b = create_tensor(tn(LLM_TENSOR_ATTN_RESID_SCALE, "bias", i), {n_embd}, 0);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_router}, 0);
layer.ffn_gate_inp_b = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "bias", i), {n_router}, 0);
z.router_eda = create_tensor(tn(LLM_TENSOR_ROUTER_EDA, "weight", i), {n_router}, i == 0 ? TENSOR_NOT_REQUIRED : 0);
z.router_norm = create_tensor(tn(LLM_TENSOR_ROUTER_NORM, "weight", i), {n_router}, 0);
z.router_fc1 = create_tensor(tn(LLM_TENSOR_ROUTER_FC1, "weight", i), {n_router, n_router}, 0);
z.router_fc1_b = create_tensor(tn(LLM_TENSOR_ROUTER_FC1, "bias", i), {n_router}, 0);
z.router_fc2 = create_tensor(tn(LLM_TENSOR_ROUTER_FC2, "weight", i), {n_router, n_router}, 0);
z.router_fc2_b = create_tensor(tn(LLM_TENSOR_ROUTER_FC2, "bias", i), {n_router}, 0);
// one extra class for the skip expert
z.router_out = create_tensor(tn(LLM_TENSOR_ROUTER_OUT, "weight", i), {n_router, n_expert + 1}, 0);
layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert + 1}, 0);
layer.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", i), {n_embd, 2 * n_ff_exp, n_expert}, 0);
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
z.ffn_res_hs = create_tensor(tn(LLM_TENSOR_FFN_RES_SCALE_HS, "weight", i), {n_embd}, 0);
layer.ffn_resid_scale = create_tensor(tn(LLM_TENSOR_FFN_RESID_SCALE, "weight", i), {n_embd}, 0);
layer.ffn_resid_scale_b = create_tensor(tn(LLM_TENSOR_FFN_RESID_SCALE, "bias", i), {n_embd}, 0);
}
}
std::unique_ptr<llm_graph_context> llama_model_zaya1::build_arch_graph(const llm_graph_params & params) const {
return std::make_unique<graph>(*this, params);
}
static ggml_tensor * zaya1_residual_scale(ggml_context * ctx, ggml_tensor * hs, ggml_tensor * res,
ggml_tensor * s_hs, ggml_tensor * s_res, ggml_tensor * b) {
return ggml_add(ctx, ggml_mul(ctx, hs, s_hs), ggml_add(ctx, ggml_mul(ctx, res, s_res), b));
}
// grouped causal conv1d with kernel 2, as one mul_mat per tap broadcast over the groups
// x: {n_embd_head * n_group, n_t + 1, n_seqs}, w: {n_embd_head (in), n_embd_head (out), n_group, 2}
// returns {n_embd_head * n_group, n_t, n_seqs} with the bias b added
static ggml_tensor * zaya1_conv_grouped(ggml_context * ctx, ggml_tensor * x, ggml_tensor * w, ggml_tensor * b) {
const int64_t n_embd_head = w->ne[0];
const int64_t n_group = w->ne[2];
const int64_t n_t = x->ne[1] - 1;
const int64_t n_seqs = x->ne[2];
GGML_ASSERT(w->ne[3] == 2);
x = ggml_reshape_4d(ctx, x, n_embd_head, n_group, n_t + 1, n_seqs);
x = ggml_permute(ctx, x, 0, 2, 1, 3);
ggml_tensor * x_prev = ggml_view_4d(ctx, x, n_embd_head, n_t, n_group, n_seqs, x->nb[1], x->nb[2], x->nb[3], 0);
ggml_tensor * x_cur = ggml_view_4d(ctx, x, n_embd_head, n_t, n_group, n_seqs, x->nb[1], x->nb[2], x->nb[3], x->nb[1]);
ggml_tensor * w_prev = ggml_view_3d(ctx, w, n_embd_head, n_embd_head, n_group, w->nb[1], w->nb[2], 0);
ggml_tensor * w_cur = ggml_view_3d(ctx, w, n_embd_head, n_embd_head, n_group, w->nb[1], w->nb[2], w->nb[3]);
ggml_tensor * y = ggml_add(ctx, ggml_mul_mat(ctx, w_prev, x_prev), ggml_mul_mat(ctx, w_cur, x_cur));
y = ggml_add(ctx, ggml_permute(ctx, y, 0, 2, 1, 3), ggml_reshape_2d(ctx, b, n_embd_head, n_group));
return ggml_reshape_3d(ctx, y, n_embd_head * n_group, n_t, n_seqs);
}
ggml_tensor * llama_model_zaya1::graph::build_cca_block(const llama_model & model, ggml_tensor * cur,
ggml_tensor * inp_pos, llm_graph_input_mem_hybrid * inp, int il) {
const auto & layer = model.layers[il];
const auto & z = layer.zaya1;
const auto * mctx_cur = inp->get_recr()->mctx;
const uint32_t kv_head = mctx_cur->get_head();
const int64_t n_seq_tokens = ubatch.n_seq_tokens;
const int64_t n_seqs = ubatch.n_seqs;
GGML_ASSERT(n_seqs != 0);
GGML_ASSERT(ubatch.equal_seqs());
GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);
const int64_t n_embd_head = hparams.n_embd_head_k();
const int64_t n_head_kv = hparams.n_head_kv(il);
const int64_t n_q = n_embd_head * n_head;
const int64_t n_qk = hparams.n_cca_qk;
const int64_t n_v_half = hparams.n_embd_v_gqa(il) / 2;
const int64_t n_hist = 2 * (hparams.ssm_d_conv - 1);
auto [q_proj, k_proj, v_raw] = build_qkv(layer, cur,
n_embd_head, n_head, n_embd_head, n_head_kv, n_embd_head, n_head_kv, il, false);
ggml_tensor * qk_raw = ggml_concat(ctx0, q_proj, k_proj, 0);
cb(qk_raw, "cca_qk_raw", il);
cb(v_raw, "cca_v_raw", il);
const size_t es = ggml_element_size(qk_raw);
const int64_t n_rep = n_head / n_head_kv;
ggml_tensor * q_raw = ggml_view_4d(ctx0, qk_raw, n_embd_head, n_rep, n_head_kv, n_tokens,
n_embd_head * es, n_embd_head * n_rep * es, qk_raw->nb[1], 0);
ggml_tensor * k_raw = ggml_view_4d(ctx0, qk_raw, n_embd_head, 1, n_head_kv, n_tokens,
n_embd_head * es, n_embd_head * es, qk_raw->nb[1], n_q * es);
ggml_tensor * q_res = ggml_scale(ctx0, ggml_add(ctx0, q_raw, k_raw), 0.5f); // {n_embd_head, n_rep, n_head_kv, n_tokens}
ggml_tensor * k_res = nullptr;
for (int64_t r = 0; r < n_rep; ++r) {
ggml_tensor * q_r = ggml_view_4d(ctx0, q_res, n_embd_head, 1, n_head_kv, n_tokens,
q_res->nb[1], q_res->nb[2], q_res->nb[3], r * q_res->nb[1]);
k_res = k_res ? ggml_add(ctx0, k_res, q_r) : q_r;
}
k_res = ggml_scale(ctx0, k_res, 1.0f / n_rep);
q_res = ggml_reshape_3d(ctx0, q_res, n_embd_head, n_head, n_tokens);
k_res = ggml_reshape_3d(ctx0, k_res, n_embd_head, n_head_kv, n_tokens);
// conv state: the last n_hist raw q/k columns of each sequence
ggml_tensor * conv_state = mctx_cur->get_r_l(il);
ggml_tensor * conv_rs = build_rs(inp->get_recr(), conv_state, hparams.n_embd_r(), n_seqs);
conv_rs = ggml_reshape_3d(ctx0, conv_rs, n_hist, n_qk, n_seqs);
ggml_tensor * qk = ggml_view_3d(ctx0, qk_raw, n_qk, n_seq_tokens, n_seqs, qk_raw->nb[1], n_seq_tokens * qk_raw->nb[1], 0);
qk = ggml_concat(ctx0, conv_rs, ggml_transpose(ctx0, qk), 0); // {n_hist + n_seq_tokens, n_qk, n_seqs}
// delayed value state: v_del of the previous token
ggml_tensor * v_state = mctx_cur->get_s_l(il);
ggml_tensor * v_rs = build_rs(inp->get_recr(), v_state, hparams.n_embd_s(), n_seqs);
ggml_tensor * v_del_ext = ggml_concat(ctx0,
ggml_reshape_3d(ctx0, v_rs, n_v_half, 1, n_seqs),
ggml_view_3d(ctx0, v_raw, n_v_half, n_seq_tokens, n_seqs, v_raw->nb[1], n_seq_tokens * v_raw->nb[1], n_v_half * es), 1); // {n_v_half, n_seq_tokens + 1, n_seqs}
{
const size_t conv_row = ggml_row_size(conv_state->type, n_hist * n_qk);
const size_t v_row = ggml_row_size(v_state->type, n_v_half);
ggml_build_forward_expand(gf, ggml_cpy(ctx0,
ggml_view_3d(ctx0, qk, n_hist, n_qk, n_seqs, qk->nb[1], qk->nb[2], n_seq_tokens * ggml_element_size(qk)),
ggml_view_2d(ctx0, conv_state, n_hist * n_qk, n_seqs, conv_state->nb[1], kv_head * conv_row)));
ggml_build_forward_expand(gf, ggml_cpy(ctx0,
ggml_view_2d(ctx0, v_del_ext, n_v_half, n_seqs, v_del_ext->nb[2], n_seq_tokens * v_del_ext->nb[1]),
ggml_view_2d(ctx0, v_state, n_v_half, n_seqs, v_state->nb[1], kv_head * v_row)));
}
// depthwise conv, then grouped conv
ggml_tensor * qk_conv = ggml_ssm_conv(ctx0, qk, layer.ssm_conv1d); // {n_qk, n_seq_tokens + 1, n_seqs}
qk_conv = ggml_add(ctx0, qk_conv, layer.ssm_conv1d_b);
qk_conv = zaya1_conv_grouped(ctx0, qk_conv, z.cca_conv_grp, z.cca_conv_grp_b); // {n_qk, n_seq_tokens, n_seqs}
qk_conv = ggml_reshape_2d(ctx0, qk_conv, n_qk, n_tokens);
cb(qk_conv, "cca_qk_conv", il);
ggml_tensor * Qcur = ggml_view_3d(ctx0, qk_conv, n_embd_head, n_head, n_tokens,
n_embd_head * ggml_element_size(qk_conv), qk_conv->nb[1], 0);
ggml_tensor * Kcur = ggml_view_3d(ctx0, qk_conv, n_embd_head, n_head_kv, n_tokens,
n_embd_head * ggml_element_size(qk_conv), qk_conv->nb[1], n_q * ggml_element_size(qk_conv));
Qcur = ggml_add(ctx0, Qcur, q_res);
Kcur = ggml_add(ctx0, Kcur, k_res);
// L2 norm to sqrt(head_dim), then per kv head key temperature
const float head_scale = sqrtf((float) n_embd_head);
Qcur = ggml_scale(ctx0, ggml_l2_norm(ctx0, Qcur, 1e-6f), head_scale);
Kcur = ggml_scale(ctx0, ggml_l2_norm(ctx0, Kcur, 1e-6f), head_scale);
Kcur = ggml_mul(ctx0, Kcur, ggml_reshape_2d(ctx0, z.cca_temp, 1, n_head_kv));
cb(Qcur, "cca_q", il);
cb(Kcur, "cca_k", il);
Qcur = build_rope(Qcur, inp_pos, il);
Kcur = build_rope(Kcur, inp_pos, il);
// value heads: head 0 = current token, head 1 = previous token
ggml_tensor * v_prev = ggml_view_3d(ctx0, v_del_ext, n_v_half, n_seq_tokens, n_seqs,
v_del_ext->nb[1], v_del_ext->nb[2], 0);
ggml_tensor * v_cur = ggml_view_3d(ctx0, v_raw, n_v_half, n_seq_tokens, n_seqs, v_raw->nb[1], n_seq_tokens * v_raw->nb[1], 0);
ggml_tensor * Vcur = ggml_concat(ctx0, v_cur, v_prev, 0);
Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
cb(Vcur, "cca_v", il);
return build_attn(inp->get_attn(),
layer.wo, NULL, layer.wo_s,
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf((float) n_embd_head), il);
}
ggml_tensor * llama_model_zaya1::graph::build_router_moe(const llama_model & model, ggml_tensor * cur,
ggml_tensor *& router_state, int il) {
const auto & layer = model.layers[il];
const auto & z = layer.zaya1;
ggml_tensor * r = ggml_add(ctx0, build_lora_mm(layer.ffn_gate_inp, cur), layer.ffn_gate_inp_b);
if (router_state) {
r = ggml_add(ctx0, r, ggml_mul(ctx0, router_state, z.router_eda));
}
router_state = r;
r = build_norm(r, z.router_norm, NULL, LLM_NORM_RMS, il);
r = ggml_gelu_erf(ctx0, ggml_add(ctx0, build_lora_mm(z.router_fc1, r), z.router_fc1_b));
r = ggml_gelu_erf(ctx0, ggml_add(ctx0, build_lora_mm(z.router_fc2, r), z.router_fc2_b));
ggml_tensor * logits = build_lora_mm(z.router_out, r); // {n_expert + 1, n_tokens}
ggml_tensor * probs = ggml_soft_max(ctx0, logits);
cb(probs, "router_probs", il);
const int64_t n_rows = cur->ne[1];
ggml_tensor * scores = ggml_add(ctx0, probs, layer.ffn_exp_probs_b);
ggml_tensor * real = ggml_cont(ctx0, ggml_view_2d(ctx0, scores, n_expert, n_rows, scores->nb[1], 0));
ggml_tensor * skip = ggml_view_2d(ctx0, scores, 1, n_rows, scores->nb[1], n_expert * ggml_element_size(scores));
ggml_tensor * expert = ggml_argsort_top_k(ctx0, real, 1); // {1, n_rows}
ggml_tensor * best = ggml_reshape_2d(ctx0, ggml_get_rows(ctx0, ggml_reshape_3d(ctx0, real, 1, n_expert, n_rows), expert), 1, n_rows);
ggml_tensor * keep = ggml_scale_bias(ctx0, ggml_step(ctx0, ggml_sub(ctx0, skip, best)), -1.0f, 1.0f);
cb(keep, "router_keep", il);
ggml_tensor * moe = build_moe_ffn(cur,
nullptr,
nullptr,
nullptr,
layer.ffn_down_exps,
nullptr,
n_expert + 1, n_expert_used,
LLM_FFN_SILU, false,
0.0f,
LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
il,
logits,
layer.ffn_gate_up_exps,
expert);
return ggml_mul(ctx0, moe, keep);
}
llama_model_zaya1::graph::graph(const llama_model & model, const llm_graph_params & params) :
llm_graph_context(params) {
ggml_tensor * cur = build_inp_embd(model.tok_embd);
cur = ggml_mul(ctx0, ggml_add(ctx0, cur, model.res_scale_in_b), model.res_scale_in);
cb(cur, "inp_scaled", -1);
auto * inp = build_inp_mem_hybrid();
ggml_tensor * inp_pos = build_inp_pos();
ggml_tensor * inp_out_ids = build_inp_out_ids();
ggml_tensor * router_state = nullptr;
for (int il = 0; il < n_layer; ++il) {
const auto & layer = model.layers[il];
const auto & z = layer.zaya1;
res->t_layer_inp[il] = cur;
ggml_tensor * residual = cur;
cur = build_norm(cur, layer.attn_norm, NULL, LLM_NORM_RMS, il);
cur = build_cca_block(model, cur, inp_pos, inp, il);
cb(cur, "attn_out", il);
if (il == n_layer - 1 && inp_out_ids) {
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
residual = ggml_get_rows(ctx0, residual, inp_out_ids);
if (router_state) {
router_state = ggml_get_rows(ctx0, router_state, inp_out_ids);
}
}
residual = zaya1_residual_scale(ctx0, cur, residual, z.attn_res_hs, layer.attn_resid_scale, layer.attn_resid_scale_b);
cur = build_norm(residual, layer.ffn_norm, NULL, LLM_NORM_RMS, il);
cur = build_router_moe(model, cur, router_state, il);
cb(cur, "ffn_moe_out", il);
cur = zaya1_residual_scale(ctx0, cur, residual, z.ffn_res_hs, layer.ffn_resid_scale, layer.ffn_resid_scale_b);
cur = build_cvec(cur, il);
cb(cur, "l_out", il);
}
cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
cb(cur, "result_norm", -1);
res->t_embd = cur;
cur = build_lora_mm(model.output, cur);
cb(cur, "result_output", -1);
res->t_logits = cur;
}