Temporary carry of the open upstream PR https://github.com/ggml-org/llama.cpp/pull/32740 at head 808fd1c4e, its four commits (1fa93f902, d29f9d489, ff91e4f84, 808fd1c4e) squashed. Inkling's per-query bias table is applied inside flash attention instead of a dense [n_kv, n_tokens, n_head] bias tensor, which cuts Inkling's compute buffers and long-context prefill time on the RX 580. Drop this commit when the PR is merged upstream (FORK.md, "Temporary upstream carries"). One conflict, ggml-vulkan.cpp use_mask_opt: kept the fork's GCN `HSK >= 256 || HSV >= 256` clause and its comment (F2) and upstream's `!use_sparse`, added the PR's `!bias &&`. Original commit trailer: Assisted-by: Codex Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PZz44SLQvTXMyWGio6t9DZ
445 lines
21 KiB
C++
445 lines
21 KiB
C++
#include "models.h"
|
|
#include "../llama-memory-hybrid-iswa.h"
|
|
|
|
void llama_model_inkling2::load_arch_hparams(llama_model_loader & ml) {
|
|
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
|
|
ml.get_key(LLM_KV_ATTENTION_QK_CLIP, hparams.f_qk_clip);
|
|
ml.get_key(LLM_KV_ATTN_RES_BLOCK_SIZE, hparams.attn_res_block_size);
|
|
ml.get_key(LLM_KV_SHORTCONV_L_CACHE, hparams.n_shortconv_l_cache);
|
|
|
|
// relative-position bias, and the log-scaled positions of the full-attention layers
|
|
ml.get_key(LLM_KV_ATTENTION_RELATIVE_LENGTH, hparams.n_relative);
|
|
ml.get_key(LLM_KV_ATTENTION_RELATIVE_EXTENT, hparams.n_relative_extent);
|
|
ml.get_key(LLM_KV_ATTENTION_RELATIVE_CLIP, hparams.f_relative_clip);
|
|
ml.get_key(LLM_KV_ATTENTION_LOG_SCALE_FLOOR, hparams.n_log_scale_floor);
|
|
ml.get_key(LLM_KV_ATTENTION_LOG_SCALE_ALPHA, hparams.f_log_scale_alpha);
|
|
|
|
// the embedding and output tensors are padded past the vocabulary
|
|
ml.get_key(LLM_KV_VOCAB_SIZE, hparams.n_vocab_padded);
|
|
ml.get_key(LLM_KV_VOCAB_SIZE_UNPADDED, hparams.n_vocab_unpadded);
|
|
|
|
// MoE
|
|
ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);
|
|
|
|
ml.get_key(LLM_KV_ACTIVATION_SITU_BETA, hparams.situ_beta);
|
|
ml.get_key(LLM_KV_ACTIVATION_SITU_LINEAR_BETA, hparams.situ_linear_beta);
|
|
ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);
|
|
|
|
hparams.f_final_logit_softcapping = 0.0f;
|
|
ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping, false);
|
|
|
|
// sliding window
|
|
ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
|
|
ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);
|
|
|
|
uint32_t n_pattern = 0;
|
|
ml.get_arr_n(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, n_pattern);
|
|
|
|
if (n_pattern != hparams.n_layer() || !hparams.n_relative || !hparams.n_relative_extent || !hparams.n_swa ||
|
|
!hparams.attn_res_block_size || !hparams.n_log_scale_floor || hparams.n_shortconv_l_cache < 2 ||
|
|
hparams.n_shortconv_l_cache > 64 || hparams.n_relative_extent > INT32_MAX || hparams.n_swa > INT32_MAX ||
|
|
!hparams.n_vocab_unpadded || hparams.n_vocab_unpadded > hparams.n_vocab_padded) {
|
|
throw std::runtime_error("inkling2: invalid dimensions, vocabulary, or layer pattern");
|
|
}
|
|
|
|
for (float value : {hparams.f_norm_rms_eps, hparams.f_qk_clip, hparams.f_relative_clip, hparams.situ_beta, hparams.situ_linear_beta}) {
|
|
if (!std::isfinite(value) || value <= 0.0f) {
|
|
throw std::runtime_error("inkling2: normalization, clipping and scaling parameters must be finite and positive");
|
|
}
|
|
}
|
|
|
|
if (!std::isfinite(hparams.f_log_scale_alpha) || hparams.f_log_scale_alpha < 0.0f) {
|
|
throw std::runtime_error("inkling2: invalid logarithmic scale");
|
|
}
|
|
|
|
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
|
|
hparams.expert_router_f32 = true;
|
|
hparams.expert_weights_norm_eps = 1e-20f;
|
|
|
|
// no RoPE, the relative-position bias carries the positions
|
|
hparams.rope_pattern.fill(false);
|
|
|
|
// the convolution state keeps K, V, the attention output and the FFN output of the widest layer
|
|
uint64_t n_channels_max = 0;
|
|
|
|
for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
|
|
if (!hparams.n_head_kv(il) || hparams.n_head(il) % hparams.n_head_kv(il) ||
|
|
hparams.n_embd_head_k(il) != hparams.n_embd_head_v(il) || !hparams.n_expert_used(il) ||
|
|
hparams.n_expert_used(il) > hparams.n_expert || !hparams.n_ff_exp(il)) {
|
|
throw std::runtime_error("inkling2: invalid attention or expert dimensions");
|
|
}
|
|
|
|
n_channels_max = std::max<uint64_t>(n_channels_max, 2ull * hparams.n_embd_k_gqa(il) + 2ull * hparams.n_embd);
|
|
}
|
|
|
|
if (n_channels_max * (hparams.n_shortconv_l_cache - 1) > UINT32_MAX) {
|
|
throw std::runtime_error("inkling2: convolution state is too large");
|
|
}
|
|
|
|
hparams.n_sconv_channels = n_channels_max;
|
|
|
|
type = LLM_TYPE_UNKNOWN;
|
|
}
|
|
|
|
void llama_model_inkling2::load_arch_tensors(llama_model_loader &) {
|
|
LLAMA_LOAD_LOCALS;
|
|
|
|
if (hparams.n_vocab_unpadded != vocab.n_tokens()) {
|
|
throw std::runtime_error("inkling2: tokenizer size must equal the unpadded vocabulary size");
|
|
}
|
|
|
|
const int64_t n_vocab_padded = hparams.n_vocab_padded;
|
|
const int64_t n_rel = hparams.n_relative;
|
|
const int64_t n_conv = hparams.n_shortconv_l_cache;
|
|
|
|
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab_padded}, 0);
|
|
tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {n_embd}, 0);
|
|
|
|
// output
|
|
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab_padded}, 0);
|
|
prec_policy.set_acc(output, GGML_PREC_F32);
|
|
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
|
|
output_res_query = create_tensor(tn(LLM_TENSOR_OUTPUT_RES_QUERY, "weight"), {n_embd}, 0);
|
|
|
|
for (int i = 0; i < n_layer_phys(); ++i) {
|
|
auto & layer = layers[i];
|
|
|
|
const int64_t n_head = hparams.n_head(i);
|
|
const int64_t n_head_kv = hparams.n_head_kv(i);
|
|
const int64_t n_embd_head = hparams.n_embd_head_k(i);
|
|
const int64_t n_ff_exp = hparams.n_ff_exp(i);
|
|
const int64_t n_extent = hparams.is_swa(i) ? hparams.n_swa : hparams.n_relative_extent;
|
|
|
|
// attention
|
|
layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head*n_head}, 0);
|
|
layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_head*n_head_kv}, 0);
|
|
layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_head*n_head_kv}, 0);
|
|
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head*n_head, n_embd}, 0);
|
|
layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_embd_head*n_head}, 0);
|
|
|
|
layer.attn_q_clip = create_tensor(tn(LLM_TENSOR_ATTN_Q_CLIP, "weight", i), {n_embd_head, n_head}, 0);
|
|
layer.attn_sink_proj = create_tensor(tn(LLM_TENSOR_ATTN_SINK_PROJ, "weight", i), {n_embd, n_head}, 0);
|
|
|
|
// relative-position bias
|
|
layer.attn_rel = create_tensor(tn(LLM_TENSOR_ATTN_REL, "weight", i), {n_embd, n_rel*n_head}, 0);
|
|
layer.attn_rel_bias = create_tensor(tn(LLM_TENSOR_ATTN_REL_BIAS, "weight", i), {n_rel, n_extent}, 0);
|
|
layer.attn_rel_clip = create_tensor(tn(LLM_TENSOR_ATTN_REL_CLIP, "weight", i), {n_rel, n_head}, 0);
|
|
|
|
// short convolutions
|
|
layer.attn_k_conv = create_tensor(tn(LLM_TENSOR_ATTN_K_CONV, "weight", i), {n_conv, n_embd_head*n_head_kv}, 0);
|
|
layer.attn_v_conv = create_tensor(tn(LLM_TENSOR_ATTN_V_CONV, "weight", i), {n_conv, n_embd_head*n_head_kv}, 0);
|
|
layer.attn_post_conv = create_tensor(tn(LLM_TENSOR_ATTN_POST_CONV, "weight", i), {n_conv, n_embd}, 0);
|
|
layer.ffn_post_conv = create_tensor(tn(LLM_TENSOR_FFN_POST_CONV, "weight", i), {n_conv, n_embd}, 0);
|
|
|
|
if (hparams.is_swa(i)) {
|
|
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
|
|
} else {
|
|
// full-attention layers: separate Q, K and V input norms, and input-dependent taps for the K and V convolutions
|
|
layer.attn_q_inp_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_INP_NORM, "weight", i), {n_embd}, 0);
|
|
layer.attn_k_inp_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_INP_NORM, "weight", i), {n_embd}, 0);
|
|
layer.attn_v_inp_norm = create_tensor(tn(LLM_TENSOR_ATTN_V_INP_NORM, "weight", i), {n_embd}, 0);
|
|
layer.attn_k_conv_proj = create_tensor(tn(LLM_TENSOR_ATTN_K_CONV_PROJ, "weight", i), {n_embd, n_conv*n_head_kv}, 0);
|
|
layer.attn_v_conv_proj = create_tensor(tn(LLM_TENSOR_ATTN_V_CONV_PROJ, "weight", i), {n_embd, n_conv*n_head_kv}, 0);
|
|
}
|
|
|
|
// block attention residuals
|
|
if (i > 0) {
|
|
layer.attn_res_query = create_tensor(tn(LLM_TENSOR_ATTN_RES_QUERY, "weight", i), {n_embd, hparams.is_swa(i) ? 1 : 3}, 0);
|
|
}
|
|
layer.ffn_res_query = create_tensor(tn(LLM_TENSOR_FFN_RES_QUERY, "weight", i), {n_embd}, 0);
|
|
|
|
// MoE
|
|
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
|
|
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
|
|
layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED);
|
|
layer.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", i), {n_embd, 2*n_ff_exp, n_expert}, 0);
|
|
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
|
|
}
|
|
}
|
|
|
|
std::unique_ptr<llm_graph_context> llama_model_inkling2::build_arch_graph(const llm_graph_params & params) const {
|
|
return std::make_unique<graph>(params);
|
|
}
|
|
|
|
// scale each row down to an L2 norm of at most threshold, after the optional per-channel scale
|
|
ggml_tensor * llama_model_inkling2::graph::clip(ggml_tensor * cur, ggml_tensor * scale, float threshold) {
|
|
if (scale) {
|
|
cur = ggml_mul(ctx0, cur, scale);
|
|
}
|
|
|
|
ggml_tensor * norm = ggml_sqrt(ctx0, ggml_sum_rows(ctx0, ggml_sqr(ctx0, cur)));
|
|
norm = ggml_scale_bias(ctx0, norm, 1.0f, 1e-6f);
|
|
|
|
ggml_tensor * factor = ggml_div(ctx0, ggml_fill(ctx0, norm, threshold), norm);
|
|
|
|
return ggml_mul(ctx0, cur, ggml_clamp(ctx0, factor, 0.0f, 1.0f));
|
|
}
|
|
|
|
// attention over the depth: mix the earlier block outputs and cur with learned queries
|
|
ggml_tensor * llama_model_inkling2::graph::mix(const std::vector<ggml_tensor *> & blocks, ggml_tensor * cur, ggml_tensor * query) {
|
|
ggml_tensor * states = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens);
|
|
for (ggml_tensor * block : blocks) {
|
|
states = ggml_concat(ctx0, ggml_reshape_3d(ctx0, block, n_embd, 1, n_tokens), states, 1);
|
|
}
|
|
states = ggml_rms_norm(ctx0, states, 1e-6f);
|
|
|
|
ggml_tensor * queries = ggml_repeat(ctx0, query, ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_embd, query->ne[1], n_tokens));
|
|
|
|
ggml_tensor * scores = ggml_mul_mat(ctx0, states, queries);
|
|
scores = ggml_soft_max_ext(ctx0, scores, nullptr, 1.0f/sqrtf(n_embd), 0.0f);
|
|
|
|
return ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, states)), scores);
|
|
}
|
|
|
|
// causal short convolution with a residual; dynamic adds input-dependent taps per head
|
|
// the last n_conv - 1 inputs of each sequence live in the recurrent state at channel offset `offset`
|
|
ggml_tensor * llama_model_inkling2::graph::convolve(
|
|
ggml_tensor * cur,
|
|
ggml_tensor * weight,
|
|
ggml_tensor * dynamic,
|
|
ggml_tensor * state,
|
|
ggml_tensor * history,
|
|
llm_graph_input_rs * inp,
|
|
int64_t offset) {
|
|
const int64_t n_conv = hparams.n_shortconv_l_cache;
|
|
const int64_t n_channels = cur->ne[0];
|
|
const int64_t n_seq_tokens = ubatch.n_seq_tokens;
|
|
const int64_t n_seqs = ubatch.n_seqs;
|
|
|
|
ggml_tensor * past = ggml_view_3d(ctx0, history, n_conv - 1, n_channels, n_seqs, history->nb[1], history->nb[2], offset*history->nb[1]);
|
|
|
|
ggml_tensor * x = ggml_transpose(ctx0, ggml_reshape_3d(ctx0, cur, n_channels, n_seq_tokens, n_seqs));
|
|
x = ggml_concat(ctx0, past, x, 0);
|
|
|
|
build_conv_state_store(inp, state, x, n_conv - 1, offset*(n_conv - 1)*ggml_element_size(state));
|
|
|
|
ggml_tensor * out = ggml_ssm_conv(ctx0, x, weight);
|
|
out = ggml_reshape_2d(ctx0, out, n_channels, n_tokens);
|
|
|
|
if (dynamic) {
|
|
const int64_t n_heads = dynamic->ne[0]/n_conv;
|
|
|
|
dynamic = ggml_reshape_3d(ctx0, dynamic, n_conv, n_heads, n_tokens);
|
|
|
|
for (int64_t lag = 0; lag < n_conv; ++lag) {
|
|
ggml_tensor * tap = ggml_view_3d(ctx0, dynamic, 1, n_heads, n_tokens, dynamic->nb[1], dynamic->nb[2], lag*dynamic->nb[0]);
|
|
ggml_tensor * window = ggml_view_3d(ctx0, x, n_seq_tokens, n_channels, n_seqs, x->nb[1], x->nb[2], (n_conv - 1 - lag)*x->nb[0]);
|
|
|
|
window = ggml_cont(ctx0, ggml_transpose(ctx0, window));
|
|
window = ggml_reshape_3d(ctx0, window, n_channels/n_heads, n_heads, n_tokens);
|
|
|
|
ggml_tensor * term = ggml_reshape_2d(ctx0, ggml_mul(ctx0, window, tap), n_channels, n_tokens);
|
|
|
|
out = ggml_add(ctx0, out, term);
|
|
}
|
|
}
|
|
|
|
return ggml_scale(ctx0, ggml_add(ctx0, cur, out), 1.0f/sqrtf((dynamic ? 2 : 1)*n_conv + 1));
|
|
}
|
|
|
|
// mix a cross-pass message into a looped layer input: (1 - forget)*cur + gain*message, both gated on cur and message
|
|
ggml_tensor * llama_model_inkling2::graph::control(ggml_tensor * cur, ggml_tensor * message, const loop_layer & loop, int ctrl) {
|
|
ggml_tensor * gate = ggml_concat(ctx0, ggml_rms_norm(ctx0, cur, 1e-6f), ggml_rms_norm(ctx0, message, 1e-6f), 0);
|
|
gate = build_lora_mm(loop.ctrl_down[ctrl], gate);
|
|
|
|
ggml_tensor * gain = ggml_tanh(ctx0, build_lora_mm(loop.ctrl_gain[ctrl], gate));
|
|
ggml_tensor * forget = ggml_tanh(ctx0, build_lora_mm(loop.ctrl_forget[ctrl], gate));
|
|
|
|
return ggml_add(ctx0, ggml_sub(ctx0, cur, ggml_mul(ctx0, forget, cur)), ggml_mul(ctx0, gain, message));
|
|
}
|
|
|
|
ggml_tensor * llama_model_inkling2::graph::build_inputs() {
|
|
inp = build_inp_mem_hybrid_iswa_rel(hparams.n_relative_extent, hparams.n_swa);
|
|
recr = inp->get_recr();
|
|
|
|
ggml_tensor * inp_pos = build_inp_pos();
|
|
inp_out_ids = build_inp_out_ids();
|
|
|
|
// log-scaled positions: 1 + alpha*log(max(1, (pos + 1)/floor)), multiplies q and r of the full-attention layers
|
|
tau = ggml_scale_bias(ctx0, ggml_cast(ctx0, inp_pos, GGML_TYPE_F32), 1.0f, 1.0f);
|
|
tau = ggml_log(ctx0, ggml_clamp(ctx0, ggml_scale(ctx0, tau, 1.0f/hparams.n_log_scale_floor), 1.0f, INFINITY));
|
|
tau = ggml_reshape_3d(ctx0, ggml_scale_bias(ctx0, tau, hparams.f_log_scale_alpha, 1.0f), 1, 1, n_tokens);
|
|
|
|
ggml_tensor * cur = build_inp_embd(model.tok_embd);
|
|
|
|
// important: do not normalize raw embeddings input (i.e. encoded image embeddings)
|
|
if (ubatch.token) {
|
|
cur = build_norm(cur, model.tok_norm, nullptr, LLM_NORM_RMS, -1);
|
|
}
|
|
|
|
return cur;
|
|
}
|
|
|
|
ggml_tensor * llama_model_inkling2::graph::build_layer(int il, int ip, ggml_tensor * cur, std::vector<ggml_tensor *> & blocks,
|
|
ggml_tensor * messages, const loop_layer * loop) {
|
|
GGML_ASSERT(!messages == !loop);
|
|
|
|
const auto & layer = model.layers[il];
|
|
|
|
const bool is_swa = hparams.is_swa(il);
|
|
const int64_t n_head = hparams.n_head(il);
|
|
const int64_t n_head_kv = hparams.n_head_kv(il);
|
|
const int64_t n_embd_head = hparams.n_embd_head_k(il);
|
|
const int64_t n_embd_k_gqa = n_embd_head*n_head_kv;
|
|
|
|
// the message of controller ctrl, [n_embd, n_tokens]
|
|
auto message = [&](int ctrl) {
|
|
return ggml_cont(ctx0, ggml_view_2d(ctx0, messages, n_embd, n_tokens, messages->nb[2], ctrl*messages->nb[1]));
|
|
};
|
|
|
|
ggml_tensor * state = recr->mctx->get_r_l(il);
|
|
ggml_tensor * history = build_rs(recr, state, hparams.n_embd_r(), ubatch.n_seqs);
|
|
history = ggml_reshape_3d(ctx0, history, hparams.n_shortconv_l_cache - 1, hparams.n_sconv_channels, ubatch.n_seqs);
|
|
|
|
ggml_tensor * inpSA = cur;
|
|
|
|
// attention inputs: SWA layers mix one, full-attention layers one each for Q, K and V
|
|
ggml_tensor * inputs[3] = {cur, cur, cur};
|
|
if (layer.attn_res_query) {
|
|
ggml_tensor * mixed = mix(blocks, cur, layer.attn_res_query);
|
|
for (int j = 0; j < 3; ++j) {
|
|
inputs[j] = ggml_cont(ctx0, ggml_view_2d(ctx0, mixed, n_embd, n_tokens, mixed->nb[2], (is_swa ? 0 : j)*mixed->nb[1]));
|
|
}
|
|
}
|
|
|
|
if (messages) {
|
|
// the messages make the Q, K and V inputs of a SWA layer differ
|
|
for (int j = 0; j < 3; ++j) {
|
|
inputs[j] = control(inputs[j], message(LOOP_CTRL_Q + j), *loop, LOOP_CTRL_Q + j);
|
|
}
|
|
}
|
|
|
|
if (is_swa && messages) {
|
|
for (int j = 0; j < 3; ++j) {
|
|
inputs[j] = build_norm(inputs[j], layer.attn_norm, nullptr, LLM_NORM_RMS, il);
|
|
}
|
|
} else if (is_swa) {
|
|
inputs[0] = build_norm(inputs[0], layer.attn_norm, nullptr, LLM_NORM_RMS, il);
|
|
inputs[1] = inputs[2] = inputs[0];
|
|
} else {
|
|
inputs[0] = build_norm(inputs[0], layer.attn_q_inp_norm, nullptr, LLM_NORM_RMS, il);
|
|
inputs[1] = build_norm(inputs[1], layer.attn_k_inp_norm, nullptr, LLM_NORM_RMS, il);
|
|
inputs[2] = build_norm(inputs[2], layer.attn_v_inp_norm, nullptr, LLM_NORM_RMS, il);
|
|
}
|
|
|
|
// self-attention
|
|
{
|
|
ggml_tensor * Qcur = build_lora_mm(layer.wq, inputs[0]);
|
|
ggml_tensor * Kcur = convolve(build_lora_mm(layer.wk, inputs[1]), layer.attn_k_conv,
|
|
is_swa ? nullptr : build_lora_mm(layer.attn_k_conv_proj, inputs[1]), state, history, recr, 0);
|
|
ggml_tensor * Vcur = convolve(build_lora_mm(layer.wv, inputs[2]), layer.attn_v_conv,
|
|
is_swa ? nullptr : build_lora_mm(layer.attn_v_conv_proj, inputs[2]), state, history, recr, n_embd_k_gqa);
|
|
ggml_tensor * Rcur = build_lora_mm(layer.attn_rel, inputs[0]);
|
|
|
|
Qcur = clip(ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), layer.attn_q_clip, hparams.f_qk_clip);
|
|
Kcur = clip(ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), nullptr, hparams.f_qk_clip);
|
|
Vcur = ggml_rms_norm(ctx0, ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens), hparams.f_norm_rms_eps);
|
|
Rcur = clip(ggml_reshape_3d(ctx0, Rcur, hparams.n_relative, n_head, n_tokens), layer.attn_rel_clip, hparams.f_relative_clip);
|
|
|
|
if (!is_swa) {
|
|
Qcur = ggml_mul(ctx0, Qcur, tau);
|
|
Rcur = ggml_mul(ctx0, Rcur, tau);
|
|
}
|
|
|
|
cb(Qcur, "Qcur", il);
|
|
cb(Kcur, "Kcur", il);
|
|
cb(Vcur, "Vcur", il);
|
|
cb(Rcur, "Rcur", il);
|
|
|
|
ggml_tensor * sinks = build_lora_mm(layer.attn_sink_proj, inputs[0]);
|
|
|
|
ggml_tensor * table = ggml_mul_mat(ctx0, ggml_pad(ctx0, layer.attn_rel_bias, 0, 1, 0, 0), Rcur);
|
|
|
|
// the sink logit t is per query: exp(s - t)/(1 + sum(exp(s - t))) = exp(s)/(exp(t) + sum(exp(s))), with a zero sink
|
|
table = ggml_sub(ctx0, table, ggml_reshape_3d(ctx0, sinks, 1, n_head, n_tokens));
|
|
|
|
sinks = ggml_fill(ctx0, ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_head), 0.0f);
|
|
|
|
cur = build_attn(inp,
|
|
nullptr, nullptr, nullptr,
|
|
Qcur, Kcur, Vcur, table, sinks, 1.0f/sqrtf(n_embd_head), il);
|
|
|
|
cur = ggml_mul(ctx0, cur, ggml_sigmoid(ctx0, build_lora_mm(layer.wqkv_gate, inputs[0])));
|
|
cur = convolve(build_lora_mm(layer.wo, cur), layer.attn_post_conv, nullptr, state, history, recr, 2*n_embd_k_gqa);
|
|
cb(cur, "attn_out", il);
|
|
}
|
|
|
|
// the first layer of each block starts a new residual stream, the earlier one joins the mixed blocks
|
|
if (ip % hparams.attn_res_block_size == 0) {
|
|
blocks.push_back(inpSA);
|
|
} else {
|
|
cur = ggml_add(ctx0, inpSA, cur);
|
|
}
|
|
|
|
ggml_tensor * ffn_inp = cur;
|
|
cb(ffn_inp, "ffn_inp", il);
|
|
|
|
// MoE
|
|
cur = ggml_reshape_2d(ctx0, mix(blocks, cur, layer.ffn_res_query), n_embd, n_tokens);
|
|
if (messages) {
|
|
cur = control(cur, message(LOOP_CTRL_FFN), *loop, LOOP_CTRL_FFN);
|
|
}
|
|
cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
|
|
|
|
cur = build_moe_ffn(cur,
|
|
layer.ffn_gate_inp,
|
|
nullptr,
|
|
nullptr,
|
|
layer.ffn_down_exps,
|
|
layer.ffn_exp_probs_b,
|
|
hparams.n_expert,
|
|
hparams.n_expert_used(il),
|
|
LLM_FFN_SITU, true,
|
|
hparams.expert_weights_scale,
|
|
LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,
|
|
il,
|
|
nullptr,
|
|
layer.ffn_gate_up_exps);
|
|
|
|
cur = convolve(cur, layer.ffn_post_conv, nullptr, state, history, recr, 2*n_embd_k_gqa + n_embd);
|
|
cb(cur, "ffn_out", il);
|
|
|
|
cur = ggml_add(ctx0, ffn_inp, cur);
|
|
cb(cur, "l_out", il);
|
|
|
|
return cur;
|
|
}
|
|
|
|
void llama_model_inkling2::graph::build_output(ggml_tensor * cur, const std::vector<ggml_tensor *> & blocks) {
|
|
cur = ggml_reshape_2d(ctx0, mix(blocks, cur, model.output_res_query), n_embd, n_tokens);
|
|
cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1);
|
|
|
|
if (inp_out_ids) {
|
|
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
|
|
}
|
|
|
|
cb(cur, "result_norm", -1);
|
|
res->t_embd = cur;
|
|
|
|
// lm_head
|
|
cur = build_lora_mm(model.output, cur);
|
|
|
|
if (hparams.f_final_logit_softcapping != 0.0f) {
|
|
const float cap = hparams.f_final_logit_softcapping;
|
|
|
|
cur = ggml_scale(ctx0, ggml_tanh(ctx0, ggml_scale(ctx0, cur, 1.0f/cap)), cap);
|
|
}
|
|
|
|
// drop the padded vocabulary
|
|
cur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, hparams.n_vocab_unpadded, cur->ne[1], cur->nb[1], 0));
|
|
|
|
cb(cur, "result_output", -1);
|
|
res->t_logits = cur;
|
|
}
|
|
|
|
llama_model_inkling2::graph::graph(const llm_graph_params & params) : llm_graph_context(params) {
|
|
ggml_tensor * cur = build_inputs();
|
|
|
|
std::vector<ggml_tensor *> blocks;
|
|
for (int il = 0; il < n_layer; ++il) {
|
|
cur = build_layer(il, il, cur, blocks);
|
|
}
|
|
|
|
build_output(cur, blocks);
|
|
}
|