* dflash : move the DSpark markov head and backbone graphs to dspark.cpp Move-only: build_dspark_markov_head, graph_dsv4, graph_dsv41 and graph_mistral_large4 move verbatim. build_inp_target_features and build_dspark_markov_head become static members of llama_model_dflash so both files can use them. * dflash : detect the head and the backbone once, split the loaders per file load_arch_hparams sets llama_model_dflash::variant (DFlash, DFlash2, DSpark, LiliCorr) and ::backbone (base, DSV4, DSV4.1, Mistral Large 4). The head, backbone and base layer loaders become member helpers in dflash.cpp, dflash2.cpp, lilicorr.cpp and dspark.cpp. build_arch_graph dispatches on the backbone. The DFlash2 and LiliCorr loaders create the DFlash2 conv tensors through a helper in dflash2.cpp, right after their head tensors instead of in the base layer loop; the other tensors keep their creation order. The DFlash2 loader drops its copy of the conv group size check, which master never reached. * dflash : share the injection input, token embeddings and lm head lookups build_inject_inp (target features + fc + encoder norm), get_tok_embd and get_lm_head replace the copies in the base, DSV4, DSV4.1 and Mistral Large 4 graphs. build_inp_target_features is file-static again. * dflash : split the DFlash decoder into a graph per head graph_base creates the shared inputs and provides build_inject, build_inp_tokens, build_layers and build_logits. graph_dflash (dflash.cpp), graph_dflash2 (dflash2.cpp), graph_dspark (dspark.cpp) and graph_lilicorr (lilicorr.cpp) assemble them and add their head; the selector, scorer and markov head builders move to the file of their graph, and the DFlash2 local conv run by build_layers to dflash2.cpp. build_arch_graph dispatches on the backbone, then on the head. The op order and the tensor names are unchanged. * speculative : split the DFlash driver into a class per head common_speculative_impl_draft_dflash stays the DFlash implementation and becomes the base of draft_dspark, draft_dflash2 and draft_lilicorr. Its protected constructor only reads the draft metadata; init() then sets up the contexts with the options of the draft (draft length, backend sampling, device features, masked h_nextn). Each derived class overrides draft() with the path of its head from the former draft(). DFlash2 also overrides begin() to reseed its generator per request, and LiliCorr overrides process() to store the context features of each injected chunk, which its noise block carries as its state. The factory keeps the current routing: draft-dspark picks DSpark, draft-dflash picks DFlash2 or LiliCorr from the draft metadata and DFlash otherwise. draft-dspark with a DFlash2 or LiliCorr draft now runs the DSpark reader instead of a mix of both. sample_from_anchor is logged by DSpark only. * speculative : add the draft-dflash2 and draft-lilicorr types draft-dflash2 and draft-lilicorr name the DFlash2 and LiliCorr drafts, which draft-dflash keeps running too: it picks them from the draft metadata. draft-dflash2 and draft-lilicorr must match the draft model, draft-dspark rejects a DFlash2 or LiliCorr draft, a model that is not a DFlash draft is rejected, and a run uses at most one DFlash family type; these errors leave speculative decoding uninitialized. The type detected from the draft GGUF is now the concrete one, and draft-dflash2 and draft-lilicorr download the dflash sidecar. * docs : describe the DFlash2 and LiliCorr drafts Add the DFlash2 and LiliCorr sections and the new types, and explain that draft-dflash also runs DFlash2 and LiliCorr drafts. * speculative : drop the unused DFlash is_mrope Nothing reads it since the migration to common_batch (#29385), which picks the number of M-RoPE position rows from the rope type of the draft. * doc: update spec dec * address review comments
874 lines
39 KiB
C++
874 lines
39 KiB
C++
#include "models.h"
|
|
|
|
#include "llama-impl.h"
|
|
#include "llama-kv-cache.h"
|
|
#include "llama-kv-cache-iswa.h"
|
|
|
|
// DeepSeek-V4 DSpark backbone: stages are full DSV4 blocks, uniform sliding window (the draft KV ring)
|
|
void llama_model_dflash::load_hparams_dsv4(llama_model_loader & ml) {
|
|
ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q);
|
|
ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
|
|
ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);
|
|
ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);
|
|
ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale);
|
|
ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm);
|
|
ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func);
|
|
ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp, hparams.n_layer_all);
|
|
if (!ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp, hparams.n_layer_all, 0)) {
|
|
hparams.swiglu_clamp_shexp = hparams.swiglu_clamp_exp;
|
|
}
|
|
ml.get_key(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, hparams.dsv4_o_group_count);
|
|
ml.get_key(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, hparams.dsv4_o_lora_rank);
|
|
ml.get_key(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, hparams.dsv4_hc_sinkhorn_iters);
|
|
ml.get_key(LLM_KV_HYPER_CONNECTION_EPSILON, hparams.dsv4_hc_eps);
|
|
ml.get_arr(LLM_KV_ATTENTION_COMPRESS_RATIOS, hparams.dsv4_compress_ratios, false);
|
|
ml.get_key(LLM_KV_HYPER_CONNECTION_SINGLE_PASS, hparams.dsv4_hc_single_pass, false);
|
|
|
|
GGML_ASSERT(hparams.dsv4_o_group_count > 0); // avoid div by zero
|
|
|
|
if (hparams.expert_gating_func != LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS) {
|
|
throw std::runtime_error("DSpark DSV4 draft expects sqrtsoftplus MoE scoring");
|
|
}
|
|
for (uint32_t il = 0; il < hparams.n_layer_all; ++il) {
|
|
if (hparams.dsv4_compress_ratios[il] != 0) {
|
|
throw std::runtime_error("DSpark DSV4 draft expects uncompressed attention on all stages");
|
|
}
|
|
}
|
|
|
|
GGML_ASSERT(hparams.n_swa > 0);
|
|
if (hparams.dsv4_hc_single_pass) {
|
|
// DeepSeek-V4.1 DSpark: block queries see the main-KV ring plus the block via the mask row of the
|
|
// first query, so that row's SWA window must reach window_size positions before the anchor
|
|
hparams.n_swa += 1;
|
|
}
|
|
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
|
|
hparams.set_swa_pattern(0);
|
|
for (uint32_t il = 0; il < hparams.n_layer_all; ++il) {
|
|
hparams.is_swa_impl[il] = true;
|
|
}
|
|
hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;
|
|
hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
|
|
}
|
|
|
|
// Mistral Large 4 DSpark: dense MLA stages, one sliding window, plain YaRN
|
|
void llama_model_dflash::load_hparams_mistral_large4(llama_model_loader & ml) {
|
|
ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA, hparams.n_embd_head_k_mla_impl);
|
|
ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl);
|
|
ml.get_key(LLM_KV_ROPE_SCALING_YARN_LOG_MUL, hparams.rope_yarn_log_mul, false);
|
|
ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
|
|
|
|
if (hparams.n_swa == 0) {
|
|
throw std::runtime_error("DSpark Mistral Large 4 draft expects a sliding window");
|
|
}
|
|
if (hparams.n_embd_head_k_full != hparams.n_lora_kv + hparams.n_rot()) {
|
|
throw std::runtime_error("DSpark Mistral Large 4 draft expects key_length == kv_lora_rank + rope dim");
|
|
}
|
|
|
|
// same as DSV4.1 DSpark: block uses the window of the anchor
|
|
hparams.n_swa += 1;
|
|
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
|
|
hparams.set_swa_pattern(0);
|
|
for (uint32_t il = 0; il < hparams.n_layer_all; ++il) {
|
|
hparams.is_swa_impl[il] = true;
|
|
}
|
|
hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;
|
|
hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
|
|
}
|
|
|
|
// DSpark = DFlash + a semi-autoregressive Markov head and Confidence head
|
|
void llama_model_dflash::load_tensors_dspark(int64_t n_vocab_draft) {
|
|
LLAMA_LOAD_LOCALS;
|
|
|
|
const int64_t dspark_markov_rank = ml->get_tensor_meta("markov_w1.weight")->ne[0];
|
|
|
|
dspark_markov_w1 = create_tensor(tn(LLM_TENSOR_DSPARK_MARKOV_W1, "weight"), { dspark_markov_rank, n_vocab }, 0);
|
|
dspark_markov_w2 = create_tensor(tn(LLM_TENSOR_DSPARK_MARKOV_W2, "weight"), { dspark_markov_rank, n_vocab_draft }, 0);
|
|
dspark_markov_w2_s = create_tensor(tn(LLM_TENSOR_DSPARK_MARKOV_W2, "scale"), { 1 }, TENSOR_NOT_REQUIRED);
|
|
|
|
dspark_conf_proj = create_tensor(tn(LLM_TENSOR_DSPARK_CONF_PROJ, "weight"), { n_embd + dspark_markov_rank, 1 }, TENSOR_NOT_REQUIRED);
|
|
dspark_conf_proj_b = create_tensor(tn(LLM_TENSOR_DSPARK_CONF_PROJ, "bias"), { 1 }, TENSOR_NOT_REQUIRED);
|
|
|
|
dspark_joint_proj = create_tensor(tn(LLM_TENSOR_DSPARK_JOINT_PROJ, "weight"), { n_embd + 2*dspark_markov_rank, 3*dspark_markov_rank }, TENSOR_NOT_REQUIRED);
|
|
dspark_joint_proj_b = create_tensor(tn(LLM_TENSOR_DSPARK_JOINT_PROJ, "bias"), { 3*dspark_markov_rank }, dspark_joint_proj ? 0 : TENSOR_NOT_REQUIRED);
|
|
|
|
LLAMA_LOG_INFO("%s: DFlash with DSpark markov head (rank = %lld)\n", __func__, (long long) dspark_markov_rank);
|
|
}
|
|
|
|
void llama_model_dflash::load_tensors_layers_dsv4() {
|
|
LLAMA_LOAD_LOCALS;
|
|
|
|
const int64_t q_lora_rank = hparams.n_lora_q;
|
|
const int64_t n_ff_exp = hparams.n_ff_exp();
|
|
const int64_t n_expert_shared = hparams.n_expert_shared;
|
|
const int64_t n_embd_head = hparams.n_embd_head_k();
|
|
const int64_t o_groups = hparams.dsv4_o_group_count;
|
|
const int64_t o_lora_rank = hparams.dsv4_o_lora_rank;
|
|
const int64_t hc_mult = hparams.dsv4_hc_mult;
|
|
const int64_t hc_dim = hc_mult * n_embd;
|
|
const int64_t hc_mix_dim = (2 + hc_mult) * hc_mult;
|
|
|
|
// single-pass mHC: collapse with the last FFN pre-mix, no hc_head
|
|
if (!hparams.dsv4_hc_single_pass) {
|
|
hc_head_fn = create_tensor(tn(LLM_TENSOR_HC_HEAD_FN, "weight"), {hc_dim, hc_mult}, 0);
|
|
hc_head_base = create_tensor(tn(LLM_TENSOR_HC_HEAD_BASE, "weight"), {hc_mult}, 0);
|
|
hc_head_scale = create_tensor(tn(LLM_TENSOR_HC_HEAD_SCALE, "weight"), {1}, 0);
|
|
}
|
|
|
|
for (int i = 0; i < n_layer; ++i) {
|
|
auto & layer = layers[i];
|
|
|
|
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
|
|
layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head}, 0);
|
|
layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, "weight", i), {n_embd, q_lora_rank}, 0);
|
|
layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, "weight", i), {q_lora_rank}, 0);
|
|
layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head * n_embd_head}, 0);
|
|
layer.wkv = create_tensor(tn(LLM_TENSOR_ATTN_KV, "weight", i), {n_embd, n_embd_head}, 0);
|
|
layer.attn_kv_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_NORM, "weight", i), {n_embd_head}, 0);
|
|
layer.wo_a = create_tensor(tn(LLM_TENSOR_ATTN_OUT_A, "weight", i), {n_head * n_embd_head / o_groups, o_lora_rank, o_groups}, TENSOR_ALLOW_RESHAPE);
|
|
layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_B, "weight", i), {o_groups * o_lora_rank, n_embd}, 0);
|
|
|
|
layer.hc_attn_fn = create_tensor(tn(LLM_TENSOR_HC_ATTN_FN, "weight", i), {hc_dim, hc_mix_dim}, 0);
|
|
layer.hc_attn_base = create_tensor(tn(LLM_TENSOR_HC_ATTN_BASE, "weight", i), {hc_mix_dim}, 0);
|
|
layer.hc_attn_scale = create_tensor(tn(LLM_TENSOR_HC_ATTN_SCALE, "weight", i), {3}, 0);
|
|
layer.hc_ffn_fn = create_tensor(tn(LLM_TENSOR_HC_FFN_FN, "weight", i), {hc_dim, hc_mix_dim}, 0);
|
|
layer.hc_ffn_base = create_tensor(tn(LLM_TENSOR_HC_FFN_BASE, "weight", i), {hc_mix_dim}, 0);
|
|
layer.hc_ffn_scale = create_tensor(tn(LLM_TENSOR_HC_FFN_SCALE, "weight", i), {3}, 0);
|
|
|
|
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
|
|
layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0);
|
|
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
|
|
|
|
layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0);
|
|
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
|
|
layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0);
|
|
|
|
layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0);
|
|
layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_exp * n_expert_shared, n_embd }, 0);
|
|
layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0);
|
|
}
|
|
}
|
|
|
|
void llama_model_dflash::load_tensors_layers_mistral_large4() {
|
|
LLAMA_LOAD_LOCALS;
|
|
|
|
const int64_t kv_lora_rank = hparams.n_lora_kv;
|
|
const int64_t n_embd_head_k_mla = hparams.n_embd_head_k_mla();
|
|
const int64_t n_embd_head_v_mla = hparams.n_embd_head_v_mla();
|
|
const int64_t n_embd_head_nope = n_embd_head_k_mla - n_rot;
|
|
|
|
dspark_mask_embd = create_tensor(tn(LLM_TENSOR_DSPARK_MASK_EMBD, "weight"), { n_embd, hparams.dflash_block_size - 1 }, 0);
|
|
|
|
for (int i = 0; i < n_layer; ++i) {
|
|
auto & layer = layers[i];
|
|
|
|
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }, 0);
|
|
layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_head * n_embd_head_k_mla }, 0);
|
|
layer.wkv_a_mqa = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_MQA, "weight", i), { n_embd, kv_lora_rank + n_rot }, 0);
|
|
layer.attn_kv_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_NORM, "weight", i), { kv_lora_rank }, 0);
|
|
layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K_B, "weight", i), { n_embd_head_nope, kv_lora_rank, n_head }, 0);
|
|
layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V_B, "weight", i), { kv_lora_rank, n_embd_head_v_mla, n_head }, 0);
|
|
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_head * n_embd_head_v_mla, n_embd }, 0);
|
|
|
|
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), { n_embd }, 0);
|
|
layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), { n_embd, n_ff }, 0);
|
|
layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd }, 0);
|
|
layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), { n_embd, n_ff }, 0);
|
|
}
|
|
}
|
|
|
|
// DSpark (DFlash + Markov & Confidence head): Markov bias on the draft logits, chained per block position
|
|
static void build_dspark_markov_head(llm_graph_context & g, const llama_model & model, ggml_tensor * tokens, ggml_tensor * hidden = nullptr) {
|
|
ggml_context * ctx0 = g.ctx0;
|
|
auto & res = g.res;
|
|
|
|
ggml_tensor * w1 = model.dspark_markov_w1;
|
|
ggml_tensor * w2 = model.dspark_markov_w2;
|
|
GGML_ASSERT(w1 && w2 && "DSpark markov weights not loaded");
|
|
|
|
// confidence head is optional
|
|
const bool has_conf = model.dspark_conf_proj != nullptr;
|
|
|
|
// rnn head: gated state carried across block positions
|
|
const bool is_rnn = model.dspark_joint_proj != nullptr;
|
|
GGML_ASSERT(!is_rnn || hidden != nullptr);
|
|
|
|
ggml_tensor * base = res->t_logits; // [n_vocab, n_tokens]
|
|
const int64_t n_vocab = base->ne[0];
|
|
const int64_t n_tok = base->ne[1];
|
|
|
|
const auto it = model.gguf_kv.find("dflash.block_size");
|
|
GGML_ASSERT(it != model.gguf_kv.end() && "DSpark draft requires 'dflash.block_size' in GGUF metadata");
|
|
const int64_t block_size = std::stoi(it->second);
|
|
GGML_ASSERT(block_size > 0);
|
|
|
|
// bonus anchor (SpecForge exports): slot 0 is a bonus token, not a prediction slot
|
|
const auto it_anchor = model.gguf_kv.find("dflash.sample_from_anchor");
|
|
const bool sample_from_anchor = it_anchor == model.gguf_kv.end() || it_anchor->second == "true";
|
|
const int64_t i_draft_beg = sample_from_anchor ? 0 : 1;
|
|
|
|
const int64_t n_blocks = g.ubatch.n_seqs_unq;
|
|
GGML_ASSERT(n_blocks > 0 && n_tok % n_blocks == 0 && "DSpark markov head requires equal-size blocks");
|
|
// runtime tokens per block in this ubatch (anchor + drafted positions), bounded by training block_size
|
|
const int64_t block_drafts = n_tok / n_blocks;
|
|
if (block_drafts > block_size) {
|
|
return;
|
|
}
|
|
|
|
// anchor (committed last) token of every block: token 0 of each block, i.e. a strided view
|
|
const size_t token_stride = (size_t) block_drafts * tokens->nb[0];
|
|
const size_t base_stride = (size_t) block_drafts * base->nb[1];
|
|
|
|
ggml_tensor * prev = ggml_view_2d(ctx0, tokens, 1, n_blocks, token_stride, 0);
|
|
prev = ggml_cont_1d(ctx0, prev, n_blocks);
|
|
|
|
ggml_tensor * cat = nullptr;
|
|
ggml_tensor * cat_conf = nullptr;
|
|
|
|
if (!sample_from_anchor) {
|
|
// bonus anchor slot: pass the logits through unbiased, pad the (unread) confidence column
|
|
cat = ggml_cont(ctx0, ggml_view_2d(ctx0, base, n_vocab, n_blocks, base_stride, 0));
|
|
if (has_conf) {
|
|
cat_conf = ggml_sigmoid(ctx0, ggml_view_2d(ctx0, base, 1, n_blocks, base_stride, 0));
|
|
}
|
|
}
|
|
|
|
ggml_tensor * state = nullptr; // [R, n_blocks]
|
|
|
|
// TODO: the in-graph chain is greedy (argmax); sampling params affect only the final
|
|
// token pick, not the Markov conditioning path
|
|
for (int64_t i = i_draft_beg; i < block_drafts; ++i) {
|
|
ggml_tensor * w1_prev = ggml_get_rows(ctx0, w1, prev); // [R, n_blocks]
|
|
ggml_tensor * markov = w1_prev;
|
|
|
|
if (is_rnn) {
|
|
const int64_t rank = w1_prev->ne[0];
|
|
if (state == nullptr) {
|
|
state = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, rank, n_blocks), 0.0f);
|
|
}
|
|
|
|
ggml_tensor * hidden_i = ggml_view_2d(ctx0, hidden, hidden->ne[0], n_blocks,
|
|
(size_t) block_drafts * hidden->nb[1], i*hidden->nb[1]);
|
|
ggml_tensor * joint = ggml_concat(ctx0, ggml_concat(ctx0, state, w1_prev, 0), ggml_cont(ctx0, hidden_i), 0);
|
|
joint = ggml_add(ctx0, ggml_mul_mat(ctx0, model.dspark_joint_proj, joint), model.dspark_joint_proj_b); // [3R, n_blocks]
|
|
|
|
const auto chunk = [&](int64_t k) {
|
|
return ggml_cont(ctx0, ggml_view_2d(ctx0, joint, rank, n_blocks, joint->nb[1], k*rank*ggml_element_size(joint)));
|
|
};
|
|
ggml_tensor * gate = ggml_sigmoid(ctx0, chunk(0));
|
|
ggml_tensor * cand = ggml_tanh(ctx0, chunk(1));
|
|
|
|
// state = gate * state + (1 - gate) * cand
|
|
state = ggml_add(ctx0, cand, ggml_mul(ctx0, gate, ggml_sub(ctx0, state, cand)));
|
|
markov = ggml_tanh(ctx0, chunk(2));
|
|
}
|
|
|
|
ggml_tensor * bias = g.build_lora_mm(w2, markov, model.dspark_markov_w2_s); // [n_vocab_draft, n_blocks]
|
|
if (model.d2t) {
|
|
// reduced draft vocab: scatter the bias to the target rows (base is -inf on the others)
|
|
const int64_t n_draft_vocab = bias->ne[0];
|
|
ggml_tensor * full = ggml_fill(ctx0, ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 1, n_vocab, n_blocks), 0.0f);
|
|
bias = ggml_set_rows(ctx0, full,
|
|
ggml_reshape_3d(ctx0, bias, 1, n_draft_vocab, n_blocks),
|
|
ggml_reshape_3d(ctx0, model.d2t, n_draft_vocab, 1, 1));
|
|
bias = ggml_reshape_2d(ctx0, bias, n_vocab, n_blocks);
|
|
}
|
|
|
|
// position i of every block: strided view [n_vocab, n_blocks]
|
|
ggml_tensor * base_i = ggml_view_2d(ctx0, base, n_vocab, n_blocks, base_stride, i*base->nb[1]);
|
|
ggml_tensor * col = ggml_add(ctx0, base_i, bias);
|
|
|
|
cat = cat ? ggml_concat(ctx0, cat, col, 1) : col;
|
|
|
|
if (has_conf) {
|
|
// confidence head input: predicts per-position acceptance
|
|
ggml_tensor * conf_inp = res->t_embd; // [n_embd, n_tok]
|
|
// conf(i) = sigmoid(conf_proj . [conf_inp(i); markov_w1[prev(i)]] + b) -- [1, n_blocks]
|
|
ggml_tensor * conf_inp_i = ggml_view_2d(ctx0, conf_inp, conf_inp->ne[0], n_blocks,
|
|
(size_t) block_drafts * conf_inp->nb[1], i*conf_inp->nb[1]);
|
|
ggml_tensor * feat = ggml_concat(ctx0, ggml_cont(ctx0, conf_inp_i), markov, 0);
|
|
ggml_tensor * conf = ggml_mul_mat(ctx0, model.dspark_conf_proj, feat);
|
|
if (model.dspark_conf_proj_b) {
|
|
conf = ggml_add(ctx0, conf, model.dspark_conf_proj_b);
|
|
}
|
|
conf = ggml_sigmoid(ctx0, conf);
|
|
|
|
cat_conf = cat_conf ? ggml_concat(ctx0, cat_conf, conf, 1) : conf;
|
|
}
|
|
|
|
if (i + 1 < block_drafts) {
|
|
prev = ggml_argmax(ctx0, col);
|
|
}
|
|
}
|
|
|
|
// cat is position-major; restore ubatch block-major order
|
|
ggml_tensor * out = ggml_reshape_3d(ctx0, cat, n_vocab, n_blocks, block_drafts);
|
|
out = ggml_cont(ctx0, ggml_permute(ctx0, out, 0, 2, 1, 3)); // [n_vocab, block_drafts, n_blocks]
|
|
out = ggml_reshape_2d(ctx0, out, n_vocab, n_tok);
|
|
|
|
if (has_conf) {
|
|
ggml_tensor * conf = ggml_reshape_3d(ctx0, cat_conf, 1, n_blocks, block_drafts);
|
|
conf = ggml_cont(ctx0, ggml_permute(ctx0, conf, 0, 2, 1, 3));
|
|
conf = ggml_reshape_2d(ctx0, conf, 1, n_tok);
|
|
|
|
// note: broadcast the [1, n_tok] confidences to n_embd-wide rows to be able to reuse `llama_get_embeddings_nextn`
|
|
conf = ggml_repeat(ctx0, conf, res->t_embd);
|
|
res->t_h_nextn = conf;
|
|
ggml_build_forward_expand(g.gf, conf);
|
|
}
|
|
|
|
res->t_logits = out;
|
|
ggml_build_forward_expand(g.gf, out);
|
|
}
|
|
|
|
llama_model_dflash::graph_dspark::graph_dspark(const llm_graph_params & params) : graph_base(params) {
|
|
ASSERT_EMBD_OR_TOKEN(ubatch);
|
|
if (ubatch.embd || ubatch.embd_row) {
|
|
build_inject();
|
|
return;
|
|
}
|
|
|
|
ggml_tensor * inp_tokens = build_inp_tokens();
|
|
ggml_tensor * cur = build_layers(inp_tokens);
|
|
|
|
build_logits(cur);
|
|
|
|
// bias the draft logits with the Markov head
|
|
build_dspark_markov_head(*this, model, inp_tokens);
|
|
}
|
|
|
|
// DSV4 DSpark decoder, dual-mode by batch type (see graph_base):
|
|
// * embd batch -> project main_x through each stage's wkv and inject K into the ring cache
|
|
// * token batch -> noise block through 3 full DSV4 stages (hc + MLA + MoE), markov + confidence heads
|
|
llama_model_dflash::graph_dsv4::graph_dsv4(const llm_graph_params & params) :
|
|
llama_model_deepseek4::graph(params, no_build{}) {
|
|
const int64_t n_embd_head = hparams.n_embd_head_k();
|
|
const int64_t n_embd_head_rope = hparams.n_rot();
|
|
const int64_t n_embd_head_nope = n_embd_head - n_embd_head_rope;
|
|
|
|
ggml_tensor * inp_pos = build_inp_pos();
|
|
|
|
llm_graph_input_attn_k_iswa * inp_attn = build_attn_inp_k_iswa();
|
|
|
|
// KV cache injection: fused target features from the encoder
|
|
ASSERT_EMBD_OR_TOKEN(ubatch);
|
|
if (ubatch.embd || ubatch.embd_row) {
|
|
ggml_tensor * inp_g = build_inject_inp(*this, model);
|
|
cb(inp_g, "inp_g_embeddings", -1);
|
|
|
|
for (int il = 0; il < n_layer; ++il) {
|
|
const auto & layer = model.layers[il];
|
|
|
|
// main-track KV: kv_norm(wkv(main_x)) with rope on the trailing dims, same
|
|
// rope parameters as the uncompressed layers in build_attention_impl
|
|
ggml_tensor * kv = build_lora_mm(layer.wkv, inp_g);
|
|
kv = build_norm(kv, layer.attn_kv_norm, nullptr, LLM_NORM_RMS, il);
|
|
kv = ggml_reshape_3d(ctx0, kv, n_embd_head, 1, n_tokens);
|
|
|
|
kv = ggml_rope_ext(ctx0, kv, inp_pos, nullptr, n_embd_head_rope, rope_type, 0,
|
|
freq_base, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
|
kv = ggml_rope_set_offset(kv, n_embd_head_nope);
|
|
cb(kv, "kv_injected", il);
|
|
|
|
if (inp_attn->self_k_rot_swa[il]) {
|
|
kv = llama_mul_mat_hadamard(ctx0, kv, inp_attn->self_k_rot_swa[il]);
|
|
}
|
|
ggml_build_forward_expand(gf, inp_attn->mctx->get_swa()->cpy_k(ctx0, kv, inp_attn->get_k_idxs_swa(), il));
|
|
}
|
|
|
|
res->t_embd = inp_g;
|
|
|
|
ggml_build_forward_expand(gf, inp_g);
|
|
return;
|
|
}
|
|
|
|
ggml_tensor * tok_embd = get_tok_embd(*this, model);
|
|
|
|
auto inp = std::make_unique<llm_graph_input_embd>(n_embd);
|
|
|
|
inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
|
|
ggml_set_input(inp->tokens);
|
|
|
|
ggml_tensor * inp_tokens = inp->tokens;
|
|
|
|
ggml_tensor * inpL = ggml_get_rows(ctx0, tok_embd, inp->tokens);
|
|
cb(inpL, "inp_noise_embd", -1);
|
|
|
|
res->add_input(std::move(inp));
|
|
|
|
const int64_t hc = hparams.dsv4_hc_mult;
|
|
inpL = ggml_reshape_3d(ctx0, inpL, n_embd, 1, n_tokens);
|
|
inpL = ggml_repeat_4d(ctx0, inpL, n_embd, hc, n_tokens, 1);
|
|
cb(inpL, "hc_init", -1);
|
|
|
|
for (int il = 0; il < n_layer; ++il) {
|
|
const auto & layer = model.layers[il];
|
|
|
|
ggml_tensor * residual = inpL;
|
|
hc_pre attn_hc = build_hc_pre(inpL,
|
|
layer.hc_attn_fn,
|
|
layer.hc_attn_scale,
|
|
layer.hc_attn_base,
|
|
il);
|
|
ggml_tensor * cur = attn_hc.cur;
|
|
cb(cur, "hc_attn_pre", il);
|
|
|
|
cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
|
|
cb(cur, "attn_norm", il);
|
|
|
|
cur = build_attention(inp_attn, cur, inp_pos, il);
|
|
|
|
inpL = build_hc_post(cur, residual, attn_hc.post, attn_hc.comb, il);
|
|
cb(inpL, "hc_attn_post", il);
|
|
|
|
residual = inpL;
|
|
hc_pre ffn_hc = build_hc_pre(inpL,
|
|
layer.hc_ffn_fn,
|
|
layer.hc_ffn_scale,
|
|
layer.hc_ffn_base,
|
|
il);
|
|
cur = ffn_hc.cur;
|
|
cb(cur, "hc_ffn_pre", il);
|
|
|
|
cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
|
|
cb(cur, "ffn_norm", il);
|
|
|
|
ggml_tensor * moe_out = build_moe_ffn(cur,
|
|
layer.ffn_gate_inp,
|
|
layer.ffn_up_exps,
|
|
layer.ffn_gate_exps,
|
|
layer.ffn_down_exps,
|
|
layer.ffn_exp_probs_b,
|
|
n_expert, hparams.n_expert_used(),
|
|
LLM_FFN_SILU, hparams.expert_weights_norm,
|
|
hparams.expert_weights_scale,
|
|
(llama_expert_gating_func_type) hparams.expert_gating_func,
|
|
il);
|
|
cb(moe_out, "ffn_moe_out", il);
|
|
|
|
ggml_tensor * ffn_shexp = build_ffn(cur,
|
|
layer.ffn_up_shexp, nullptr, nullptr,
|
|
layer.ffn_gate_shexp, nullptr, nullptr,
|
|
layer.ffn_down_shexp, nullptr, nullptr,
|
|
nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il);
|
|
cb(ffn_shexp, "ffn_shexp", il);
|
|
|
|
cur = ggml_add(ctx0, moe_out, ffn_shexp);
|
|
cb(cur, "ffn_out", il);
|
|
|
|
inpL = build_hc_post(cur, residual, ffn_hc.post, ffn_hc.comb, il);
|
|
cb(inpL, "l_out", il);
|
|
}
|
|
|
|
ggml_tensor * cur = build_hc_head(inpL, model.hc_head_fn, model.hc_head_scale, model.hc_head_base);
|
|
cb(cur, "hc_head", -1);
|
|
|
|
// confidence head input: the reference scores the pre-norm collapsed hidden state
|
|
res->t_embd = cur;
|
|
|
|
cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1);
|
|
cb(cur, "result_norm", -1);
|
|
|
|
const auto [output, output_s] = get_lm_head(*this, model);
|
|
cur = build_lora_mm(output, cur, output_s);
|
|
cb(cur, "result_output", -1);
|
|
res->t_logits = cur;
|
|
|
|
if (model.dspark_markov_w1) {
|
|
build_dspark_markov_head(*this, model, inp_tokens);
|
|
}
|
|
}
|
|
|
|
// DeepSeek-V4.1 DSpark decoder (mtp.0..2), dual-mode by batch type like graph_dsv4:
|
|
// * embd batch -> writes every stage's kv_norm(wkv(main_proj(target features))) into its window ring
|
|
// * token batch -> [anchor, noise...] block through the stages, single-pass mHC, no per-head q norm
|
|
// window of a block = window of its first query, see n_swa
|
|
|
|
llama_model_dflash::graph_dsv41::graph_dsv41(const llm_graph_params & params) :
|
|
llama_model_deepseek41::graph(params, no_build{}) {
|
|
const int64_t n_embd_head = hparams.n_embd_head_k();
|
|
const int64_t n_embd_head_rope = hparams.n_rot();
|
|
const int64_t n_embd_head_nope = n_embd_head - n_embd_head_rope;
|
|
|
|
GGML_ASSERT(hparams.dsv4_hc_single_pass);
|
|
|
|
ggml_tensor * inp_pos = build_inp_pos();
|
|
|
|
llm_graph_input_attn_k_iswa * inp_attn = build_attn_inp_k_iswa();
|
|
|
|
// ratio-0 layers: plain rope at rope_theta, no YaRN (original_seq_len 0)
|
|
const auto rope = [&](ggml_tensor * t) {
|
|
t = ggml_rope_ext(ctx0, t, inp_pos, nullptr, n_embd_head_rope, rope_type, 0,
|
|
freq_base, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
|
return ggml_rope_set_offset(t, n_embd_head_nope);
|
|
};
|
|
|
|
ASSERT_EMBD_OR_TOKEN(ubatch);
|
|
if (ubatch.embd || ubatch.embd_row) {
|
|
ggml_tensor * inp_g = build_inject_inp(*this, model);
|
|
cb(inp_g, "main_x", -1);
|
|
|
|
for (int il = 0; il < n_layer; ++il) {
|
|
const auto & layer = model.layers[il];
|
|
|
|
ggml_tensor * kv = build_lora_mm(layer.wkv, inp_g);
|
|
kv = build_norm(kv, layer.attn_kv_norm, nullptr, LLM_NORM_RMS, il);
|
|
kv = ggml_reshape_3d(ctx0, kv, n_embd_head, 1, n_tokens);
|
|
kv = rope(kv);
|
|
cb(kv, "kv_injected", il);
|
|
|
|
if (inp_attn->self_k_rot_swa[il]) {
|
|
kv = llama_mul_mat_hadamard(ctx0, kv, inp_attn->self_k_rot_swa[il]);
|
|
}
|
|
ggml_build_forward_expand(gf, inp_attn->mctx->get_swa()->cpy_k(ctx0, kv, inp_attn->get_k_idxs_swa(), il));
|
|
}
|
|
|
|
res->t_embd = inp_g;
|
|
|
|
ggml_build_forward_expand(gf, inp_g);
|
|
return;
|
|
}
|
|
|
|
ggml_tensor * tok_embd = get_tok_embd(*this, model);
|
|
|
|
auto inp = std::make_unique<llm_graph_input_embd>(n_embd);
|
|
|
|
inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
|
|
ggml_set_input(inp->tokens);
|
|
|
|
ggml_tensor * inp_tokens = inp->tokens;
|
|
|
|
ggml_tensor * inpL = ggml_get_rows(ctx0, tok_embd, inp->tokens);
|
|
cb(inpL, "inp_noise_embd", -1);
|
|
|
|
res->add_input(std::move(inp));
|
|
|
|
const int64_t hc = hparams.dsv4_hc_mult;
|
|
const int64_t n_groups = hparams.dsv4_o_group_count;
|
|
const int64_t n_heads_group = n_head / n_groups;
|
|
const int64_t o_lora_rank = hparams.dsv4_o_lora_rank;
|
|
const int64_t o_group_dim = n_heads_group*n_embd_head;
|
|
const float kq_scale = 1.0f/sqrtf(float(n_embd_head));
|
|
|
|
GGML_ASSERT(n_embd_head == n_embd_head_v);
|
|
GGML_ASSERT(n_head % n_groups == 0);
|
|
|
|
inpL = ggml_reshape_3d(ctx0, inpL, n_embd, 1, n_tokens);
|
|
inpL = ggml_repeat_4d(ctx0, inpL, n_embd, hc, n_tokens, 1);
|
|
cb(inpL, "hc_init", -1);
|
|
|
|
// the whole block shares the mask row of its first query (the anchor)
|
|
inp_attn->self_attn_mask_swa.block_uniform = true;
|
|
|
|
ggml_tensor * pre_prev = nullptr;
|
|
|
|
for (int il = 0; il < n_layer; ++il) {
|
|
const auto & layer = model.layers[il];
|
|
|
|
ggml_tensor * residual = inpL;
|
|
const hc_mixes attn_mix = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, il);
|
|
|
|
ggml_tensor * cur;
|
|
if (pre_prev == nullptr) {
|
|
// identity pre-mix of the first stage: copy 0
|
|
cur = ggml_cont(ctx0, ggml_view_2d(ctx0, inpL, n_embd, n_tokens, inpL->nb[2], 0));
|
|
} else {
|
|
cur = build_hc_pre(inpL, pre_prev, il);
|
|
}
|
|
cb(cur, "hc_attn_pre", il);
|
|
|
|
cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
|
|
cb(cur, "attn_norm", il);
|
|
|
|
// non-causal draft context: block queries see the main-KV ring plus every block key
|
|
{
|
|
ggml_tensor * qr = build_lora_mm(layer.wq_a, cur);
|
|
qr = build_norm(qr, layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il);
|
|
cb(qr, "qr_norm", il);
|
|
|
|
ggml_tensor * q = build_lora_mm(layer.wq_b, qr);
|
|
q = ggml_reshape_3d(ctx0, q, n_embd_head, n_head, n_tokens);
|
|
q = rope(q);
|
|
cb(q, "q", il);
|
|
|
|
ggml_tensor * kv = build_lora_mm(layer.wkv, cur);
|
|
kv = build_norm(kv, layer.attn_kv_norm, nullptr, LLM_NORM_RMS, il);
|
|
kv = ggml_reshape_3d(ctx0, kv, n_embd_head, 1, n_tokens);
|
|
kv = rope(kv);
|
|
cb(kv, "kv", il);
|
|
|
|
ggml_tensor * out = build_attn(inp_attn,
|
|
nullptr, nullptr, nullptr,
|
|
q, kv, kv,
|
|
nullptr, layer.attn_sinks, nullptr,
|
|
kq_scale, il);
|
|
cb(out, "attn_raw", il);
|
|
|
|
out = ggml_reshape_3d(ctx0, out, n_embd_head, n_head, n_tokens);
|
|
out = ggml_rope_ext_back(ctx0, out, inp_pos, nullptr, n_embd_head_rope, rope_type, 0,
|
|
freq_base, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
|
out = ggml_rope_set_offset(out, n_embd_head_nope);
|
|
cb(out, "attn_derope", il);
|
|
|
|
out = ggml_reshape_3d(ctx0, out, o_group_dim, n_groups, n_tokens);
|
|
out = ggml_permute(ctx0, out, 0, 2, 1, 3);
|
|
ggml_tensor * oa = ggml_mul_mat(ctx0, layer.wo_a, out);
|
|
cb(oa, "attn_wo_a", il);
|
|
oa = ggml_permute(ctx0, oa, 0, 2, 1, 3);
|
|
oa = ggml_cont_2d(ctx0, oa, o_lora_rank*n_groups, n_tokens);
|
|
|
|
cur = build_lora_mm(layer.wo_b, oa);
|
|
cb(cur, "attn_out", il);
|
|
}
|
|
|
|
inpL = build_hc_post(cur, residual, attn_mix.post, attn_mix.comb, il);
|
|
cb(inpL, "hc_attn_post", il);
|
|
|
|
residual = inpL;
|
|
const hc_mixes ffn_mix = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, il);
|
|
pre_prev = ffn_mix.pre;
|
|
|
|
cur = build_hc_pre(inpL, attn_mix.pre, il);
|
|
cb(cur, "hc_ffn_pre", il);
|
|
|
|
cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
|
|
cb(cur, "ffn_norm", il);
|
|
|
|
ggml_tensor * moe_out = build_moe_ffn(cur,
|
|
layer.ffn_gate_inp,
|
|
layer.ffn_up_exps,
|
|
layer.ffn_gate_exps,
|
|
layer.ffn_down_exps,
|
|
layer.ffn_exp_probs_b,
|
|
n_expert, hparams.n_expert_used(),
|
|
LLM_FFN_SILU, hparams.expert_weights_norm,
|
|
hparams.expert_weights_scale,
|
|
(llama_expert_gating_func_type) hparams.expert_gating_func,
|
|
il);
|
|
cb(moe_out, "ffn_moe_out", il);
|
|
|
|
ggml_tensor * ffn_shexp = build_ffn(cur,
|
|
layer.ffn_up_shexp, nullptr, nullptr,
|
|
layer.ffn_gate_shexp, nullptr, nullptr,
|
|
layer.ffn_down_shexp, nullptr, nullptr,
|
|
nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il);
|
|
cb(ffn_shexp, "ffn_shexp", il);
|
|
|
|
cur = ggml_add(ctx0, moe_out, ffn_shexp);
|
|
cb(cur, "ffn_out", il);
|
|
|
|
inpL = build_hc_post(cur, residual, ffn_mix.post, ffn_mix.comb, il);
|
|
|
|
ggml_tensor * l_out = ggml_reshape_2d(ctx0, inpL, n_embd*hc, n_tokens);
|
|
cb(l_out, "l_out", il);
|
|
ggml_build_forward_expand(gf, l_out);
|
|
}
|
|
|
|
// no hc_head: collapse with the last FFN's pre-mix
|
|
ggml_tensor * cur = build_hc_pre(inpL, pre_prev, -1);
|
|
cb(cur, "hc_out", -1);
|
|
|
|
// confidence head takes the un-normed collapsed state
|
|
res->t_embd = cur;
|
|
|
|
cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1);
|
|
cb(cur, "result_norm", -1);
|
|
|
|
const auto [output, output_s] = get_lm_head(*this, model);
|
|
cur = build_lora_mm(output, cur, output_s);
|
|
cb(cur, "result_output", -1);
|
|
res->t_logits = cur;
|
|
|
|
if (model.dspark_markov_w1) {
|
|
build_dspark_markov_head(*this, model, inp_tokens);
|
|
}
|
|
}
|
|
|
|
// Mistral Large 4 DSpark, dual-mode like graph_dsv41:
|
|
// - embd batch: write latent K of target_norm(fc(features)) to the cache, no attn norm
|
|
// - token batch: decode [anchor, mask_1, ...] block, non-causal
|
|
llama_model_dflash::graph_mistral_large4::graph_mistral_large4(const llm_graph_params & params) : llm_graph_context(params) {
|
|
const int64_t kv_lora_rank = hparams.n_lora_kv;
|
|
const int64_t n_embd_head_k_mla = hparams.n_embd_head_k_mla();
|
|
const int64_t n_embd_head_v_mla = hparams.n_embd_head_v_mla();
|
|
const int64_t n_embd_head_nope = n_embd_head_k_mla - n_rot;
|
|
|
|
const float kq_scale = 1.0f/sqrtf(float(n_embd_head_k_mla));
|
|
|
|
ggml_tensor * inp_pos = build_inp_pos();
|
|
|
|
llm_graph_input_attn_k_iswa * inp_attn = build_attn_inp_k_iswa();
|
|
|
|
// no LeRoPE in the draft
|
|
const auto rope = [&](ggml_tensor * t) {
|
|
return ggml_rope_ext(ctx0, t, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
ext_factor, attn_factor, beta_fast, beta_slow);
|
|
};
|
|
|
|
const auto build_kv = [&](ggml_tensor * x, int il) -> std::pair<ggml_tensor *, ggml_tensor *> {
|
|
const auto & layer = model.layers[il];
|
|
|
|
ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, layer.wkv_a_mqa, x);
|
|
cb(kv_cmpr_pe, "kv_cmpr_pe", il);
|
|
|
|
ggml_tensor * kv_cmpr = ggml_view_2d(ctx0, kv_cmpr_pe, kv_lora_rank, n_tokens,
|
|
ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_rot), 0);
|
|
kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);
|
|
kv_cmpr = ggml_reshape_3d(ctx0, kv_cmpr, kv_lora_rank, 1, n_tokens);
|
|
cb(kv_cmpr, "kv_cmpr", il);
|
|
|
|
ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_cmpr_pe, n_rot, 1, n_tokens,
|
|
ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_rot),
|
|
ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_rot),
|
|
ggml_row_size(kv_cmpr_pe->type, kv_lora_rank));
|
|
k_pe = rope(k_pe);
|
|
cb(k_pe, "k_pe", il);
|
|
|
|
return { ggml_concat(ctx0, kv_cmpr, k_pe, 0), kv_cmpr };
|
|
};
|
|
|
|
ASSERT_EMBD_OR_TOKEN(ubatch);
|
|
if (ubatch.embd || ubatch.embd_row) {
|
|
ggml_tensor * inp_g = build_inject_inp(*this, model);
|
|
cb(inp_g, "main_x", -1);
|
|
|
|
for (int il = 0; il < n_layer; ++il) {
|
|
ggml_tensor * kv = build_kv(inp_g, il).first;
|
|
cb(kv, "kv_injected", il);
|
|
|
|
if (inp_attn->self_k_rot_swa[il]) {
|
|
kv = llama_mul_mat_hadamard(ctx0, kv, inp_attn->self_k_rot_swa[il]);
|
|
}
|
|
ggml_build_forward_expand(gf, inp_attn->mctx->get_swa()->cpy_k(ctx0, kv, inp_attn->get_k_idxs_swa(), il));
|
|
}
|
|
|
|
res->t_embd = inp_g;
|
|
|
|
ggml_build_forward_expand(gf, inp_g);
|
|
return;
|
|
}
|
|
|
|
ggml_tensor * tok_embd = get_tok_embd(*this, model);
|
|
|
|
auto inp = std::make_unique<llm_graph_input_embd>(n_embd);
|
|
|
|
inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
|
|
ggml_set_input(inp->tokens);
|
|
|
|
ggml_tensor * inp_tokens = inp->tokens;
|
|
|
|
res->add_input(std::move(inp));
|
|
|
|
const int64_t n_blocks = ubatch.n_seqs_unq;
|
|
const int64_t block_drafts = n_blocks > 0 ? n_tokens / n_blocks : 0;
|
|
|
|
ggml_tensor * inpL;
|
|
if (block_drafts > 0 && n_tokens % n_blocks == 0 && block_drafts <= (int64_t) hparams.dflash_block_size) {
|
|
// slot k > 0 uses mask row k - 1
|
|
ggml_tensor * anchors = ggml_view_2d(ctx0, inp_tokens, 1, n_blocks, (size_t) block_drafts * inp_tokens->nb[0], 0);
|
|
anchors = ggml_get_rows(ctx0, tok_embd, ggml_cont_1d(ctx0, anchors, n_blocks));
|
|
inpL = ggml_reshape_3d(ctx0, anchors, n_embd, 1, n_blocks);
|
|
if (block_drafts > 1) {
|
|
ggml_tensor * masks = ggml_view_3d(ctx0, model.dspark_mask_embd, n_embd, block_drafts - 1, 1,
|
|
model.dspark_mask_embd->nb[1], model.dspark_mask_embd->nb[2], 0);
|
|
masks = ggml_repeat_4d(ctx0, ggml_cast(ctx0, masks, GGML_TYPE_F32), n_embd, block_drafts - 1, n_blocks, 1);
|
|
inpL = ggml_concat(ctx0, inpL, masks, 1);
|
|
}
|
|
inpL = ggml_reshape_2d(ctx0, inpL, n_embd, n_tokens);
|
|
} else {
|
|
// not a draft block (e.g. graph reserve)
|
|
inpL = ggml_get_rows(ctx0, tok_embd, inp_tokens);
|
|
}
|
|
cb(inpL, "inp_noise_embd", -1);
|
|
|
|
// the whole block shares the mask row of its first query (the anchor)
|
|
inp_attn->self_attn_mask_swa.block_uniform = true;
|
|
|
|
for (int il = 0; il < n_layer; ++il) {
|
|
const auto & layer = model.layers[il];
|
|
|
|
ggml_tensor * cur = build_norm(inpL, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
|
|
cb(cur, "attn_norm", il);
|
|
|
|
{
|
|
ggml_tensor * q = ggml_mul_mat(ctx0, layer.wq, cur);
|
|
cb(q, "q", il);
|
|
|
|
// {n_embd_head_nope, n_tokens, n_head}
|
|
ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_nope, n_head, n_tokens,
|
|
ggml_row_size(q->type, n_embd_head_k_mla),
|
|
ggml_row_size(q->type, n_embd_head_k_mla) * n_head, 0);
|
|
q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3);
|
|
|
|
// {kv_lora_rank, n_head, n_tokens}
|
|
ggml_tensor * q_nope_absorbed = ggml_mul_mat(ctx0, layer.wk_b, q_nope);
|
|
q_nope_absorbed = ggml_permute(ctx0, q_nope_absorbed, 0, 2, 1, 3);
|
|
cb(q_nope_absorbed, "q_nope_absorbed", il);
|
|
|
|
ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_rot, n_head, n_tokens,
|
|
ggml_row_size(q->type, n_embd_head_k_mla),
|
|
ggml_row_size(q->type, n_embd_head_k_mla) * n_head,
|
|
ggml_row_size(q->type, n_embd_head_nope));
|
|
q_pe = rope(q_pe);
|
|
cb(q_pe, "q_pe", il);
|
|
|
|
ggml_tensor * Qcur = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0);
|
|
cb(Qcur, "Qcur", il);
|
|
|
|
auto [Kcur, Vcur] = build_kv(cur, il);
|
|
cb(Kcur, "Kcur", il);
|
|
|
|
// apply wv_b after attn, so the K rotation is already undone
|
|
ggml_tensor * out = build_attn(inp_attn,
|
|
nullptr, nullptr, nullptr,
|
|
Qcur, Kcur, Vcur,
|
|
nullptr, nullptr, nullptr,
|
|
kq_scale, il);
|
|
cb(out, "attn_latent", il);
|
|
|
|
// {kv_lora_rank, n_tokens, n_head} -> {n_embd_head_v_mla, n_tokens, n_head}
|
|
out = ggml_reshape_3d(ctx0, out, kv_lora_rank, n_head, n_tokens);
|
|
out = ggml_mul_mat(ctx0, layer.wv_b, ggml_permute(ctx0, out, 0, 2, 1, 3));
|
|
out = ggml_cont_2d(ctx0, ggml_permute(ctx0, out, 0, 2, 1, 3), n_embd_head_v_mla*n_head, n_tokens);
|
|
|
|
cur = build_lora_mm(layer.wo, out, layer.wo_s);
|
|
cb(cur, "attn_out", il);
|
|
}
|
|
|
|
ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);
|
|
cb(ffn_inp, "ffn_inp", il);
|
|
|
|
cur = build_norm(ffn_inp, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
|
|
cb(cur, "ffn_norm", il);
|
|
|
|
cur = build_ffn(cur,
|
|
layer.ffn_up, nullptr, nullptr,
|
|
layer.ffn_gate, nullptr, nullptr,
|
|
layer.ffn_down, nullptr, nullptr,
|
|
nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il);
|
|
cb(cur, "ffn_out", il);
|
|
|
|
inpL = ggml_add(ctx0, cur, ffn_inp);
|
|
cb(inpL, "l_out", il);
|
|
}
|
|
|
|
ggml_tensor * cur = build_norm(inpL, model.output_norm, nullptr, LLM_NORM_RMS, -1);
|
|
cb(cur, "result_norm", -1);
|
|
res->t_embd = cur;
|
|
|
|
const auto [output, output_s] = get_lm_head(*this, model);
|
|
ggml_tensor * logits = build_lora_mm(output, cur, output_s);
|
|
|
|
// softcap does not apply to the markov bias
|
|
if (hparams.f_final_logit_softcapping) {
|
|
logits = ggml_scale(ctx0, logits, 1.0f / hparams.f_final_logit_softcapping);
|
|
logits = ggml_tanh(ctx0, logits);
|
|
logits = ggml_scale(ctx0, logits, hparams.f_final_logit_softcapping);
|
|
}
|
|
cb(logits, "result_output", -1);
|
|
res->t_logits = logits;
|
|
|
|
if (model.dspark_markov_w1) {
|
|
build_dspark_markov_head(*this, model, inp_tokens, cur);
|
|
}
|
|
}
|