* convert : add Inkling2 text GGUF conversion Assisted-by: Codex * llama : implement Inkling2 text inference Assisted-by: Codex * tests : cover Inkling2 text support Assisted-by: Codex * conversion : dequantize packed Inkling2 experts in row chunks Decode per-layer NVFP4 and MXFP8 schemes, validate companions, and use stored scale shapes with a warning for the declared MXFP8 [32,32] grid. Reorder packed w13 rows and their scale rows together. Keep normal GGUF outtype selection and 3D expert tensors. Stream 128 rows per expert through LazyChunkedTensor. Real w13 (384x4608x4608) needs a 2.25 MiB F32 chunk instead of a 30.375 GiB expanded layer. Real-shaped sparse probes peak below 5.8 MiB of Python/NumPy chunk allocations, plus 5.2 MiB of callback metadata: budget about 16 MiB per projection including mapped pages, excluding runtime, other tensors and OS cache. Full-model chunk metadata is roughly 370 MiB. Packed synthetic GGUF weights match the independent pinned vLLM decode bit-for-bit. CPU prefill/decode logits remain within 3.1e-6 absolute error, with 100% argmax agreement. Float fixtures, original-name conversion and CPU architecture checks pass. Real checkpoint conversion remains untested. Assisted-by: Codex * convert : add Inkling2 vision mmproj support Assisted-by: Codex * mtmd : implement Inkling2 still-image encoder Assisted-by: Codex * inkling2 : scope image loading Assisted-by: Codex * inkling2 : enable flash attention with folded bias and sinks Assisted-by: Codex * cuda : route per-head flash attention masks through vec Assisted-by: Codex * mtmd : support Inkling2 temporal frame means Assisted-by: Codex * style : restore shared image comment Assisted-by: Codex * tests : enlarge Inkling2 context fixture Assisted-by: Codex * inkling2 : load per-layer expert feed-forward widths Assisted-by: Codex * tests : add Inkling2 Metal fusion baselines Assisted-by: Codex * inkling2 : group and align registry entries Assisted-by: Fable * convert : regenerate the Inkling2 tokenizer hash with the update script Assisted-by: Fable * conversion : revert the print_registered_models change Assisted-by: Fable * conversion : sort the Inkling2 model map entries Assisted-by: Fable * convert : map Inkling2 vision checkpoint names in the tensor table Assisted-by: Fable * convert : map Inkling2 text checkpoint names in the tensor table Assisted-by: Fable * tests : document the Inkling2 packed expert conversion test Assisted-by: Fable * tests : cover Inkling2 checkpoint and transformers tensor names Assisted-by: Fable * llama : place Inkling2 fields and list entries by file convention Assisted-by: Fable * gguf-py : align Inkling2 tensor mapping comments Assisted-by: Fable * mtmd : place Inkling2 entries by file convention Assisted-by: Fable * tests : place Inkling2 cases after the existing ones Assisted-by: Fable * cuda : index per-head flash attention masks in the tile and mma kernels Assisted-by: Claude Claude-Session: https://claude.ai/code/session_01GbYSza1yxgmTz8wdr6uV8Q * inkling2 : project the relative bias before the gather, drop arch branches Assisted-by: Claude Claude-Session: https://claude.ai/code/session_01GbYSza1yxgmTz8wdr6uV8Q * convert : require Inkling2 hparams, add inkling2 pre-tokenizer Assisted-by: Claude Claude-Session: https://claude.ai/code/session_01GbYSza1yxgmTz8wdr6uV8Q * mtmd : cap Inkling2 image size, drop video frame averaging Assisted-by: Claude Claude-Session: https://claude.ai/code/session_01GbYSza1yxgmTz8wdr6uV8Q * inkling2 : fix CI build, Meta device assert and Windows conversion test Assisted-by: Claude * tests : use the default sliding window for the Inkling2 test model Assisted-by: Claude * simplify mtmd part * give _v arch for vision * remove alpha channel handling, it should be a follow up * explicit rule about audio + vision models * add audio support * rm audio * rm gguf mtmd rope base * address review comment * nits * do not normalize embd from mtmd --------- Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
316 lines
18 KiB
C++
316 lines
18 KiB
C++
#include "models.h"
|
|
#include "../llama-memory-hybrid-iswa.h"
|
|
|
|
void llama_model_inkling2::load_arch_hparams(llama_model_loader & ml) {
|
|
ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);
|
|
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
|
|
ml.get_key(LLM_KV_ATTENTION_RELATIVE_LENGTH, hparams.n_relative);
|
|
ml.get_key(LLM_KV_ATTENTION_RELATIVE_EXTENT, hparams.n_relative_extent);
|
|
ml.get_key(LLM_KV_ATTENTION_QK_CLIP, hparams.f_qk_clip);
|
|
ml.get_key(LLM_KV_ATTENTION_RELATIVE_CLIP, hparams.f_relative_clip);
|
|
ml.get_key(LLM_KV_ATTENTION_LOG_SCALE_FLOOR, hparams.n_log_scale_floor);
|
|
ml.get_key(LLM_KV_ATTENTION_LOG_SCALE_ALPHA, hparams.f_log_scale_alpha);
|
|
ml.get_key(LLM_KV_ATTN_RES_BLOCK_SIZE, hparams.attn_res_block_size);
|
|
ml.get_key(LLM_KV_SHORTCONV_L_CACHE, hparams.n_shortconv_l_cache);
|
|
ml.get_key(LLM_KV_VOCAB_SIZE, hparams.n_vocab_padded);
|
|
ml.get_key(LLM_KV_VOCAB_SIZE_UNPADDED, hparams.n_vocab_unpadded);
|
|
ml.get_key(LLM_KV_ACTIVATION_SITU_BETA, hparams.situ_beta);
|
|
ml.get_key(LLM_KV_ACTIVATION_SITU_LINEAR_BETA, hparams.situ_linear_beta);
|
|
ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale);
|
|
ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping);
|
|
ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
|
|
ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);
|
|
uint32_t n_pattern = 0;
|
|
ml.get_arr_n(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, n_pattern);
|
|
if (n_pattern != hparams.n_layer() || !hparams.n_relative || !hparams.n_relative_extent || !hparams.n_swa ||
|
|
!hparams.attn_res_block_size || !hparams.n_log_scale_floor || hparams.n_shortconv_l_cache < 2 ||
|
|
hparams.n_shortconv_l_cache > 64 || hparams.n_relative_extent > INT32_MAX || hparams.n_swa > INT32_MAX ||
|
|
!hparams.n_vocab_unpadded || hparams.n_vocab_unpadded > hparams.n_vocab_padded) {
|
|
throw std::runtime_error("inkling2: invalid dimensions, vocabulary, or layer pattern");
|
|
}
|
|
for (float value : {hparams.f_norm_rms_eps, hparams.f_qk_clip, hparams.f_relative_clip, hparams.situ_beta,
|
|
hparams.situ_linear_beta, hparams.expert_weights_scale, hparams.f_final_logit_softcapping}) {
|
|
if (!std::isfinite(value) || value <= 0.0f) {
|
|
throw std::runtime_error("inkling2: normalization, clipping and scaling parameters must be finite and positive");
|
|
}
|
|
}
|
|
if (!std::isfinite(hparams.f_log_scale_alpha) || hparams.f_log_scale_alpha < 0.0f) {
|
|
throw std::runtime_error("inkling2: invalid logarithmic scale");
|
|
}
|
|
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
|
|
hparams.expert_router_f32 = true;
|
|
hparams.expert_weights_norm_eps = 1e-20f;
|
|
hparams.rope_pattern.fill(false);
|
|
uint64_t max_channels = 0;
|
|
for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
|
|
if (!hparams.n_head_kv(il) || hparams.n_head(il) % hparams.n_head_kv(il) ||
|
|
hparams.n_embd_head_k(il) != hparams.n_embd_head_v(il) || !hparams.n_expert_used(il) ||
|
|
hparams.n_expert_used(il) > hparams.n_expert || !hparams.n_ff_exp(il)) {
|
|
throw std::runtime_error("inkling2: invalid attention or expert dimensions");
|
|
}
|
|
max_channels = std::max<uint64_t>(max_channels, 2ull * hparams.n_embd_k_gqa(il) + 2ull * hparams.n_embd);
|
|
}
|
|
if (max_channels * (hparams.n_shortconv_l_cache - 1) > UINT32_MAX) {
|
|
throw std::runtime_error("inkling2: convolution state is too large");
|
|
}
|
|
hparams.n_sconv_channels = max_channels;
|
|
type = LLM_TYPE_UNKNOWN;
|
|
}
|
|
|
|
void llama_model_inkling2::load_arch_tensors(llama_model_loader &) {
|
|
LLAMA_LOAD_LOCALS;
|
|
if (hparams.n_vocab_unpadded != vocab.n_tokens()) {
|
|
throw std::runtime_error("inkling2: tokenizer size must equal the unpadded vocabulary size");
|
|
}
|
|
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, hparams.n_vocab_padded}, 0);
|
|
tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {n_embd}, 0);
|
|
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, hparams.n_vocab_padded}, 0);
|
|
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
|
|
output_res_query = create_tensor(tn(LLM_TENSOR_OUTPUT_RES_QUERY, "weight"), {n_embd}, 0);
|
|
for (int i = 0; i < n_layer; ++i) {
|
|
auto & l = layers[i];
|
|
const int64_t nh = hparams.n_head(i);
|
|
const int64_t nk = hparams.n_head_kv(i);
|
|
const int64_t hd = hparams.n_embd_head_k(i);
|
|
const int64_t nr = hparams.n_relative;
|
|
const int64_t kernel = hparams.n_shortconv_l_cache;
|
|
const int64_t extent = hparams.is_swa(i) ? hparams.n_swa : hparams.n_relative_extent;
|
|
l.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, nh*hd}, 0);
|
|
l.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, nk*hd}, 0);
|
|
l.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, nk*hd}, 0);
|
|
l.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {nh*hd, n_embd}, 0);
|
|
l.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, nh*hd}, 0);
|
|
l.attn_rel = create_tensor(tn(LLM_TENSOR_ATTN_REL, "weight", i), {n_embd, nh*nr}, 0);
|
|
l.attn_rel_bias = create_tensor(tn(LLM_TENSOR_ATTN_REL_BIAS, "weight", i), {nr, extent}, 0);
|
|
l.attn_q_clip = create_tensor(tn(LLM_TENSOR_ATTN_Q_CLIP, "weight", i), {hd, nh}, 0);
|
|
l.attn_rel_clip = create_tensor(tn(LLM_TENSOR_ATTN_REL_CLIP, "weight", i), {nr, nh}, 0);
|
|
l.attn_sink_proj = create_tensor(tn(LLM_TENSOR_ATTN_SINK_PROJ, "weight", i), {n_embd, nh}, 0);
|
|
l.attn_k_conv = create_tensor(tn(LLM_TENSOR_ATTN_K_CONV, "weight", i), {kernel, nk*hd}, 0);
|
|
l.attn_v_conv = create_tensor(tn(LLM_TENSOR_ATTN_V_CONV, "weight", i), {kernel, nk*hd}, 0);
|
|
l.attn_post_conv = create_tensor(tn(LLM_TENSOR_ATTN_POST_CONV, "weight", i), {kernel, n_embd}, 0);
|
|
l.ffn_post_conv = create_tensor(tn(LLM_TENSOR_FFN_POST_CONV, "weight", i), {kernel, n_embd}, 0);
|
|
if (hparams.is_swa(i)) {
|
|
l.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
|
|
} else {
|
|
l.attn_q_inp_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_INP_NORM, "weight", i), {n_embd}, 0);
|
|
l.attn_k_inp_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_INP_NORM, "weight", i), {n_embd}, 0);
|
|
l.attn_v_inp_norm = create_tensor(tn(LLM_TENSOR_ATTN_V_INP_NORM, "weight", i), {n_embd}, 0);
|
|
l.attn_k_conv_proj = create_tensor(tn(LLM_TENSOR_ATTN_K_CONV_PROJ, "weight", i), {n_embd, kernel*nk}, 0);
|
|
l.attn_v_conv_proj = create_tensor(tn(LLM_TENSOR_ATTN_V_CONV_PROJ, "weight", i), {n_embd, kernel*nk}, 0);
|
|
}
|
|
if (i > 0) {
|
|
l.attn_res_query = create_tensor(tn(LLM_TENSOR_ATTN_RES_QUERY, "weight", i), {n_embd, hparams.is_swa(i) ? 1 : 3}, 0);
|
|
}
|
|
l.ffn_res_query = create_tensor(tn(LLM_TENSOR_FFN_RES_QUERY, "weight", i), {n_embd}, 0);
|
|
l.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
|
|
l.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
|
|
l.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED);
|
|
l.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", i), {n_embd, 2*hparams.n_ff_exp(i), n_expert}, 0);
|
|
l.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {hparams.n_ff_exp(i), n_embd, n_expert}, 0);
|
|
}
|
|
}
|
|
|
|
std::unique_ptr<llm_graph_context> llama_model_inkling2::build_arch_graph(const llm_graph_params & params) const {
|
|
return std::make_unique<graph>(params);
|
|
}
|
|
|
|
class llm_graph_input_inkling2_pos : public llm_graph_input_i {
|
|
public:
|
|
const llama_kv_cache_context * cache[2];
|
|
ggml_tensor * distance[2];
|
|
uint32_t extent[2];
|
|
|
|
bool can_reuse(const llm_graph_params & params) override {
|
|
const auto * memory = static_cast<const llama_memory_hybrid_iswa_context *>(params.mctx);
|
|
cache[0] = memory->get_attn()->get_base();
|
|
cache[1] = memory->get_attn()->get_swa();
|
|
for (int i = 0; i < 2; ++i) {
|
|
if (distance[i]->ne[0] != cache[i]->get_n_kv() || distance[i]->ne[1] != params.ubatch.n_tokens) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
void set_input(const llama_ubatch * ubatch) override {
|
|
for (int i = 0; i < 2; ++i) {
|
|
if (distance[i]->buffer) {
|
|
cache[i]->set_input_relative_pos(distance[i], ubatch, extent[i]);
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
ggml_tensor * llama_model_inkling2::graph::clip(ggml_tensor * cur, ggml_tensor * scale, float threshold) {
|
|
if (scale) {
|
|
cur = ggml_mul(ctx0, cur, scale);
|
|
}
|
|
auto * norm = ggml_sqrt(ctx0, ggml_sum_rows(ctx0, ggml_sqr(ctx0, cur)));
|
|
norm = ggml_scale_bias(ctx0, norm, 1.0f, 1e-6f);
|
|
auto * factor = ggml_div(ctx0, ggml_fill(ctx0, norm, threshold), norm);
|
|
return ggml_mul(ctx0, cur, ggml_clamp(ctx0, factor, 0.0f, 1.0f));
|
|
}
|
|
|
|
ggml_tensor * llama_model_inkling2::graph::mix(const std::vector<ggml_tensor *> & blocks, ggml_tensor * cur, ggml_tensor * query) {
|
|
auto * states = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens);
|
|
for (auto * block : blocks) {
|
|
states = ggml_concat(ctx0, ggml_reshape_3d(ctx0, block, n_embd, 1, n_tokens), states, 1);
|
|
}
|
|
states = ggml_rms_norm(ctx0, states, 1e-6f);
|
|
auto * queries = ggml_repeat(ctx0, query, ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_embd, query->ne[1], n_tokens));
|
|
auto * scores = ggml_mul_mat(ctx0, states, queries);
|
|
scores = ggml_soft_max_ext(ctx0, scores, nullptr, 1.0f/sqrtf(n_embd), 0.0f);
|
|
return ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, states)), scores);
|
|
}
|
|
|
|
ggml_tensor * llama_model_inkling2::graph::convolve(ggml_tensor * cur, ggml_tensor * weight, ggml_tensor * dynamic,
|
|
ggml_tensor * state, ggml_tensor * history, llm_graph_input_rs * inp, int64_t offset) {
|
|
const int64_t kernel = hparams.n_shortconv_l_cache;
|
|
const int64_t channels = cur->ne[0];
|
|
const int64_t nt = ubatch.n_seq_tokens;
|
|
const int64_t ns = ubatch.n_seqs;
|
|
auto * past = ggml_view_3d(ctx0, history, kernel - 1, channels, ns, history->nb[1], history->nb[2], offset*history->nb[1]);
|
|
auto * x = ggml_transpose(ctx0, ggml_reshape_3d(ctx0, cur, channels, nt, ns));
|
|
x = ggml_concat(ctx0, past, x, 0);
|
|
build_conv_state_store(inp, state, x, kernel - 1, offset*(kernel - 1)*ggml_element_size(state));
|
|
auto * out = ggml_ssm_conv(ctx0, x, weight);
|
|
out = ggml_reshape_2d(ctx0, out, channels, n_tokens);
|
|
if (dynamic) {
|
|
const int64_t heads = dynamic->ne[0]/kernel;
|
|
dynamic = ggml_reshape_3d(ctx0, dynamic, kernel, heads, n_tokens);
|
|
for (int64_t lag = 0; lag < kernel; ++lag) {
|
|
auto * tap = ggml_view_3d(ctx0, dynamic, 1, heads, n_tokens, dynamic->nb[1], dynamic->nb[2], lag*dynamic->nb[0]);
|
|
auto * window = ggml_view_3d(ctx0, x, nt, channels, ns, x->nb[1], x->nb[2], (kernel - 1 - lag)*x->nb[0]);
|
|
window = ggml_cont(ctx0, ggml_transpose(ctx0, window));
|
|
window = ggml_reshape_3d(ctx0, window, channels/heads, heads, n_tokens);
|
|
auto * term = ggml_reshape_2d(ctx0, ggml_mul(ctx0, window, tap), channels, n_tokens);
|
|
out = ggml_add(ctx0, out, term);
|
|
}
|
|
}
|
|
return ggml_scale(ctx0, ggml_add(ctx0, cur, out), 1.0f/sqrtf((dynamic ? 2 : 1)*kernel + 1));
|
|
}
|
|
|
|
llama_model_inkling2::graph::graph(const llm_graph_params & params) : llm_graph_context(params) {
|
|
auto * inp = build_inp_mem_hybrid_iswa();
|
|
auto * attn = inp->get_attn();
|
|
auto * recr = inp->get_recr();
|
|
auto * positions = build_inp_pos();
|
|
auto * out_ids = build_inp_out_ids();
|
|
auto rel = std::make_unique<llm_graph_input_inkling2_pos>();
|
|
rel->cache[0] = attn->mctx->get_base();
|
|
rel->cache[1] = attn->mctx->get_swa();
|
|
rel->extent[0] = hparams.n_relative_extent;
|
|
rel->extent[1] = hparams.n_swa;
|
|
ggml_tensor * distances[2];
|
|
for (int i = 0; i < 2; ++i) {
|
|
distances[i] = rel->distance[i] = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, rel->cache[i]->get_n_kv(), n_tokens);
|
|
ggml_set_input(distances[i]);
|
|
}
|
|
res->add_input(std::move(rel));
|
|
auto * tau = ggml_scale_bias(ctx0, ggml_cast(ctx0, positions, GGML_TYPE_F32), 1.0f, 1.0f);
|
|
tau = ggml_log(ctx0, ggml_clamp(ctx0, ggml_scale(ctx0, tau, 1.0f/hparams.n_log_scale_floor), 1.0f, INFINITY));
|
|
tau = ggml_reshape_3d(ctx0, ggml_scale_bias(ctx0, tau, hparams.f_log_scale_alpha, 1.0f), 1, 1, n_tokens);
|
|
auto * cur = build_inp_embd(model.tok_embd);
|
|
// important: do not normalize raw embeddings input (i.e. encoded image embeddings)
|
|
if (ubatch.token) {
|
|
cur = build_norm(cur, model.tok_norm, nullptr, LLM_NORM_RMS, -1);
|
|
}
|
|
std::vector<ggml_tensor *> blocks;
|
|
for (int il = 0; il < n_layer; ++il) {
|
|
const auto & l = model.layers[il];
|
|
const bool swa = hparams.is_swa(il);
|
|
const int64_t nh = hparams.n_head(il);
|
|
const int64_t nk = hparams.n_head_kv(il);
|
|
const int64_t hd = hparams.n_embd_head_k(il);
|
|
const int64_t kv = nk*hd;
|
|
auto * state = recr->mctx->get_r_l(il);
|
|
auto * history = build_rs(recr, state, hparams.n_embd_r(), ubatch.n_seqs);
|
|
history = ggml_reshape_3d(ctx0, history, hparams.n_shortconv_l_cache - 1, hparams.n_sconv_channels, ubatch.n_seqs);
|
|
auto * residual = cur;
|
|
ggml_tensor * inputs[3] = {cur, cur, cur};
|
|
if (l.attn_res_query) {
|
|
auto * mixed = mix(blocks, cur, l.attn_res_query);
|
|
for (int j = 0; j < 3; ++j) {
|
|
inputs[j] = ggml_cont(ctx0, ggml_view_2d(ctx0, mixed, n_embd, n_tokens, mixed->nb[2], (swa ? 0 : j)*mixed->nb[1]));
|
|
}
|
|
}
|
|
if (swa) {
|
|
inputs[0] = build_norm(inputs[0], l.attn_norm, nullptr, LLM_NORM_RMS, il);
|
|
inputs[1] = inputs[2] = inputs[0];
|
|
} else {
|
|
inputs[0] = build_norm(inputs[0], l.attn_q_inp_norm, nullptr, LLM_NORM_RMS, il);
|
|
inputs[1] = build_norm(inputs[1], l.attn_k_inp_norm, nullptr, LLM_NORM_RMS, il);
|
|
inputs[2] = build_norm(inputs[2], l.attn_v_inp_norm, nullptr, LLM_NORM_RMS, il);
|
|
}
|
|
auto * q = build_lora_mm(l.wq, inputs[0]);
|
|
auto * k = convolve(build_lora_mm(l.wk, inputs[1]), l.attn_k_conv,
|
|
swa ? nullptr : build_lora_mm(l.attn_k_conv_proj, inputs[1]), state, history, recr, 0);
|
|
auto * v = convolve(build_lora_mm(l.wv, inputs[2]), l.attn_v_conv,
|
|
swa ? nullptr : build_lora_mm(l.attn_v_conv_proj, inputs[2]), state, history, recr, kv);
|
|
auto * r = build_lora_mm(l.attn_rel, inputs[0]);
|
|
q = clip(ggml_reshape_3d(ctx0, q, hd, nh, n_tokens), l.attn_q_clip, hparams.f_qk_clip);
|
|
k = clip(ggml_reshape_3d(ctx0, k, hd, nk, n_tokens), nullptr, hparams.f_qk_clip);
|
|
v = ggml_rms_norm(ctx0, ggml_reshape_3d(ctx0, v, hd, nk, n_tokens), hparams.f_norm_rms_eps);
|
|
r = clip(ggml_reshape_3d(ctx0, r, hparams.n_relative, nh, n_tokens), l.attn_rel_clip, hparams.f_relative_clip);
|
|
if (!swa) {
|
|
q = ggml_mul(ctx0, q, tau);
|
|
r = ggml_mul(ctx0, r, tau);
|
|
}
|
|
auto * mask = swa ? attn->self_attn_mask_swa.raw : attn->self_attn_mask.raw;
|
|
const int64_t streams = mask->ne[3];
|
|
const int64_t n_kv = distances[swa]->ne[0];
|
|
auto * sinks = build_lora_mm(l.attn_sink_proj, inputs[0]);
|
|
// project r on the bias table first, then gather by distance: only the gathered bias is n_kv x n_tokens x nh
|
|
auto * table = ggml_mul_mat(ctx0, ggml_pad(ctx0, l.attn_rel_bias, 0, 1, 0, 0), r);
|
|
if (cparams.flash_attn) {
|
|
// exp(s - t)/(1 + sum(exp(s - t))) = exp(s)/(exp(t) + sum(exp(s))).
|
|
table = ggml_sub(ctx0, table, ggml_reshape_3d(ctx0, sinks, 1, nh, n_tokens));
|
|
} else {
|
|
// build_attn scales the bias along with QK; FA adds the mask after scaling QK.
|
|
table = ggml_scale(ctx0, table, sqrtf(hd));
|
|
}
|
|
table = ggml_cont(ctx0, ggml_permute(ctx0, table, 1, 0, 2, 3));
|
|
auto * bias = ggml_get_rows(ctx0, table, distances[swa]);
|
|
bias = ggml_cont(ctx0, ggml_permute(ctx0, ggml_reshape_4d(ctx0, bias, nh, n_kv, n_tokens/streams, streams), 2, 0, 1, 3));
|
|
ggml_tensor * fa_mask = nullptr;
|
|
if (cparams.flash_attn) {
|
|
fa_mask = ggml_cast(ctx0, ggml_add(ctx0, bias, ggml_cast(ctx0, mask, GGML_TYPE_F32)), GGML_TYPE_F16);
|
|
sinks = ggml_fill(ctx0, ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, nh), 0.0f);
|
|
bias = nullptr;
|
|
} else {
|
|
sinks = ggml_reshape_4d(ctx0, ggml_cont(ctx0, ggml_permute(ctx0,
|
|
ggml_reshape_3d(ctx0, sinks, nh, n_tokens/streams, streams), 1, 0, 2, 3)), 1, n_tokens/streams, nh, streams);
|
|
}
|
|
cur = build_attn(attn, nullptr, nullptr, nullptr, q, k, v, bias, sinks, nullptr, 1.0f/sqrtf(hd), il, fa_mask);
|
|
cur = ggml_mul(ctx0, cur, ggml_sigmoid(ctx0, build_lora_mm(l.wqkv_gate, inputs[0])));
|
|
cur = convolve(build_lora_mm(l.wo, cur), l.attn_post_conv, nullptr, state, history, recr, 2*kv);
|
|
if (il % hparams.attn_res_block_size == 0) {
|
|
blocks.push_back(residual);
|
|
} else {
|
|
cur = ggml_add(ctx0, residual, cur);
|
|
}
|
|
residual = cur;
|
|
cur = ggml_reshape_2d(ctx0, mix(blocks, cur, l.ffn_res_query), n_embd, n_tokens);
|
|
cur = build_norm(cur, l.ffn_norm, nullptr, LLM_NORM_RMS, il);
|
|
cur = build_moe_ffn(cur, l.ffn_gate_inp, nullptr, nullptr, l.ffn_down_exps, l.ffn_exp_probs_b,
|
|
hparams.n_expert, hparams.n_expert_used(il), LLM_FFN_SITU, true, hparams.expert_weights_scale,
|
|
LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID, il, nullptr, l.ffn_gate_up_exps);
|
|
cur = convolve(cur, l.ffn_post_conv, nullptr, state, history, recr, 2*kv + n_embd);
|
|
cur = ggml_add(ctx0, residual, cur);
|
|
cb(cur, "l_out", il);
|
|
}
|
|
cur = ggml_reshape_2d(ctx0, mix(blocks, cur, model.output_res_query), n_embd, n_tokens);
|
|
cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1);
|
|
if (out_ids) {
|
|
cur = ggml_get_rows(ctx0, cur, out_ids);
|
|
}
|
|
res->t_embd = cur;
|
|
cur = build_lora_mm(model.output, cur);
|
|
ggml_prec_set_acc(cur, GGML_PREC_F32);
|
|
const float cap = hparams.f_final_logit_softcapping;
|
|
cur = ggml_scale(ctx0, ggml_tanh(ctx0, ggml_scale(ctx0, cur, 1.0f/cap)), cap);
|
|
cur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, hparams.n_vocab_unpadded, cur->ne[1], cur->nb[1], 0));
|
|
res->t_logits = cur;
|
|
cb(cur, "result_output", -1);
|
|
}
|