diff --git a/.github/workflows/models-check.yml b/.github/workflows/models-check.yml index 66a150aff..2774cefd3 100644 --- a/.github/workflows/models-check.yml +++ b/.github/workflows/models-check.yml @@ -11,7 +11,8 @@ on: 'tests/fusion/**', 'tests/test-fusion.cpp', 'tests/test-llama-archs.cpp', - 'src/llama-graph.cpp', + 'src/llama-graph*', + 'src/llama-kv-cache*', 'src/llama-model*', 'src/models/**' ] @@ -24,7 +25,8 @@ on: 'tests/fusion/**', 'tests/test-fusion.cpp', 'tests/test-llama-archs.cpp', - 'src/llama-graph.cpp', + 'src/llama-graph*', + 'src/llama-kv-cache*', 'src/llama-model*', 'src/models/**' ] @@ -145,6 +147,11 @@ jobs: rm -rf build-ci-models && mkdir -p build-ci-models ./build/bin/test-llama-archs -o build-ci-models + - name: Test KV shift + id: test_kv_shift + run: | + ./build/bin/test-llama-archs --kv-shift + - name: Test fusion id: test_fusion run: | @@ -293,6 +300,11 @@ jobs: # run: | # ./build/bin/test-fusion --models build-ci-models --device Vulkan0 --check tests/fusion/Vulkan.csv + - name: Test KV shift + id: test_kv_shift + run: | + ./build/bin/test-llama-archs --kv-shift + - name: Test archs id: test_archs run: | diff --git a/skills/code-review/SKILL.md b/skills/code-review/SKILL.md index 4c8e4bbcc..d72b46712 100644 --- a/skills/code-review/SKILL.md +++ b/skills/code-review/SKILL.md @@ -85,6 +85,7 @@ See the `add-new-model` skill and `docs/development/HOWTO-add-model.md` for the - Hparams that the model can't run correctly without must be mandatory (hard-error if missing), not read with a silent default fallback. Only genuinely-optional-across-configs values get a fallback accessor. - New/optional weight tensors (scales, etc.) must route through `build_lora_mm` and the existing helpers, matching convention - don't leave raw matmuls copied from another arch. - Don't hack RoPE with a custom sin/cos implementation. If `ggml_rope_ext` genuinely can't express it, that's an issue for discussion, not a PR. +- In `src/models/`, apply RoPE with `build_rope(cur, pos, il)`, not a direct `ggml_rope_ext`/`ggml_rope_multi` call (see #31683). Per-layer differences go in a `get_rope_params` override: the KV-cache K-shift reads the same params, so a direct call lets the graph and the shift drift apart. The remaining direct calls (deepseek4/41, dflash, sensenova-u1) are not a pattern to copy. - Test the quantized-KV path (`-ctk`/`-ctv q8_0`), not just default f16 - new speculative/attention features silently break there. - Preserve existing explanatory comments about model-specific quirks when copying code; note the provenance ("copied from X, with Y added"). - Remove dead code/branches left over from adapting a reference implementation. diff --git a/src/llama-context.cpp b/src/llama-context.cpp index a1068aae3..ea20062a6 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -2910,7 +2910,6 @@ llm_graph_params llama_context::graph_params( const llama_memory_context_i * mctx, llm_graph_type gtype) const { return { - /*.arch =*/ model.arch, /*.model =*/ &model, /*.hparams =*/ model.hparams, /*.cparams =*/ cparams, diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 47007714a..b99370e2d 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1546,7 +1546,6 @@ void llm_graph_result::set_params(const llm_graph_params & params) { llm_graph_context::llm_graph_context(const llm_graph_params & params) : model (*params.model), - arch (params.arch), hparams (params.hparams), cparams (params.cparams), ubatch (params.ubatch), @@ -1640,6 +1639,37 @@ ggml_tensor * llm_graph_context::build_hadamard_input( return x; } +static ggml_tensor * build_rope_impl(ggml_context * ctx, ggml_tensor * cur, ggml_tensor * pos, const llama_hparams & hparams, const llama_rope_params & rp) { + if (rp.n_rot == 0) { + return cur; + } + + if (rp.type & GGML_ROPE_TYPE_MROPE) { + int sections[GGML_MROPE_SECTIONS]; + std::copy(hparams.rope_sections.begin(), hparams.rope_sections.end(), sections); + + return ggml_rope_multi(ctx, cur, pos, rp.factors, + rp.n_rot, sections, rp.type, rp.n_ctx_orig, rp.freq_base, rp.freq_scale, + rp.ext_factor, rp.attn_factor, rp.beta_fast, rp.beta_slow); + } + + return ggml_rope_ext(ctx, cur, pos, rp.factors, + rp.n_rot, rp.type, rp.n_ctx_orig, rp.freq_base, rp.freq_scale, + rp.ext_factor, rp.attn_factor, rp.beta_fast, rp.beta_slow); +} + +ggml_tensor * llm_graph_context::build_rope(ggml_tensor * cur, ggml_tensor * pos, int il) const { + return build_rope_impl(ctx0, cur, pos, hparams, model.get_rope_params(cparams, il)); +} + +ggml_tensor * llm_graph_context::build_rope_indexer(ggml_tensor * cur, ggml_tensor * pos, int il) const { + const llama_rope_params rp = model.get_rope_params_indexer(cparams, il); + + ggml_tensor * res = build_rope_impl(ctx0, cur, pos, hparams, rp); + + return rp.n_rot > 0 && rp.n_offs > 0 ? ggml_rope_set_offset(res, rp.n_offs) : res; +} + ggml_tensor * llm_graph_context::build_lora_mm( ggml_tensor * w, ggml_tensor * cur, @@ -1950,6 +1980,8 @@ ggml_tensor * llm_graph_context::build_ffn( llm_ffn_op_type type_op, llm_ffn_gate_type type_gate, int il) const { + const llm_arch arch = model.arch; + // NVFP4 support is currently restricted to // 1) LORA absence (*_s would be applied after LORA residual, which is incorrect) // 2) bias absense (*_s would be applied after bias addition, which is incorrect) @@ -2233,6 +2265,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn( ggml_tensor * selected_experts_in, ggml_tensor * post_exp_norm, float post_exp_norm_eps) const { + const llm_arch arch = model.arch; const int64_t n_embd = cur->ne[0]; const int64_t n_tokens = cur->ne[1]; const bool weight_before_ffn = arch == LLM_ARCH_LLAMA4; // for llama4, we apply the sigmoid-ed weights before the FFN @@ -2658,7 +2691,7 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd, float to // mixed path (ubatch.is_mixed()): set_rows the token rows into a copy of the embd rows, with its own inputs as select branches must not share tensors // TODO: use inp->tokens and inp->embd once ggml_build_forward_select allows it - const bool has_mixed = llm_arch_supports_mixed_batch(arch) && cparams.ctx_type == LLAMA_CONTEXT_TYPE_DEFAULT; + const bool has_mixed = llm_arch_supports_mixed_batch(model.arch) && cparams.ctx_type == LLAMA_CONTEXT_TYPE_DEFAULT; const int64_t n_tok_rows = has_mixed ? llm_graph_n_tok_rows(ubatch) : 0; @@ -2884,7 +2917,7 @@ ggml_tensor * llm_graph_context::build_inp_mean() const { } ggml_tensor * llm_graph_context::build_inp_cls() const { - auto inp = std::make_unique(cparams, arch); + auto inp = std::make_unique(cparams, model.arch); auto & cur = inp->cls; @@ -3055,7 +3088,7 @@ ggml_tensor * llm_graph_context::build_attn_mha( // while for some models F16 is enough, for others it is not, so we default to F32 here ggml_prec_set_acc(kq, GGML_PREC_F32); - if (arch == LLM_ARCH_GROK) { + if (model.arch == LLM_ARCH_GROK) { // need to do the following: // multiply by attn_output_multiplier // and then : @@ -3248,6 +3281,8 @@ ggml_tensor * llm_graph_context::build_attn( ggml_tensor * v_mla, // TODO: remove float kq_scale, int il) const { + const llm_arch arch = model.arch; + GGML_ASSERT(v_mla == nullptr); if (inp->self_k_rot[il]) { @@ -3361,6 +3396,8 @@ ggml_tensor * llm_graph_context::build_attn( ggml_tensor * v_mla, float kq_scale, int il) const { + const llm_arch arch = model.arch; + // these nodes are added to the graph together so that they are not reordered // by doing so, the number of splits in the graph is reduced // expand k later to enable rope fusion which directly writes into k-v cache @@ -4185,6 +4222,8 @@ void llm_graph_context::build_pooling( ggml_tensor * cls_out, ggml_tensor * cls_out_b, ggml_tensor * cls_norm) const { + const llm_arch arch = model.arch; + if (!cparams.embeddings) { // a graph without logits still computes t_embd, so that it is never empty if (res->t_logits == nullptr && res->t_embd != nullptr) { diff --git a/src/llama-graph.h b/src/llama-graph.h index d20343163..0f8ad8ac9 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -788,8 +788,6 @@ using llm_graph_cb = std::function( model, no_alloc, hparams_lid, type_k, type_v, v_trans, offload, unified, kv_size, n_seq_max, - n_swa, swa_type, nullptr, filter_lid, nullptr, nullptr); + n_swa, swa_type, nullptr, filter_lid, nullptr, nullptr, "", true); } void llama_kv_cache_dsa::clear(bool data) { diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index 06e218ba6..a1ec67e26 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -79,9 +79,10 @@ llama_kv_cache::llama_kv_cache( const layer_filter_cb & filter, const layer_reuse_cb & reuse, const layer_share_cb & share, - const char * name_tag) : + const char * name_tag, + bool is_indexer) : model(model), hparams(hparams), v_trans(v_trans), - n_seq_max(n_seq_max), n_stream(unified ? 1 : n_seq_max), n_swa(n_swa), + n_seq_max(n_seq_max), n_stream(unified ? 1 : n_seq_max), n_swa(n_swa), is_indexer(is_indexer), attn_rot_bufs(no_alloc), swa_type(swa_type), ctxs_bufs(no_alloc), other(static_cast(mem_other)), v_cells_impl(other ? other->v_cells_impl : std::make_shared()), @@ -2089,53 +2090,69 @@ size_t llama_kv_cache::size_v_bytes() const { ggml_tensor * llama_kv_cache::build_rope_shift( const llama_cparams & cparams, ggml_context * ctx, - ggml_tensor * cur, + ggml_tensor * k, ggml_tensor * shift, ggml_tensor * rot, - ggml_tensor * factors, - float freq_base, - float freq_scale, uint32_t il) const { - const auto & n_ctx_orig = cparams.n_ctx_orig_yarn; + const llama_rope_params rp = is_indexer ? model.get_rope_params_indexer(cparams, il) : model.get_rope_params(cparams, il); - const auto & yarn_ext_factor = cparams.yarn_ext_factor; - const auto & yarn_beta_fast = cparams.yarn_beta_fast; - const auto & yarn_beta_slow = cparams.yarn_beta_slow; - const auto & yarn_attn_factor = cparams.yarn_attn_factor; - - const auto & n_rot = hparams.n_rot(il); - const auto & rope_type = hparams.rope_type == LLAMA_ROPE_TYPE_MROPE || hparams.rope_type == LLAMA_ROPE_TYPE_IMROPE + const auto rope_type = rp.type == LLAMA_ROPE_TYPE_MROPE || rp.type == LLAMA_ROPE_TYPE_IMROPE // @ngxson : this is a workaround // for M-RoPE, we want to rotate the whole vector when doing KV shift // a normal RoPE should work, we just need to use the correct ordering // ref: https://github.com/ggml-org/llama.cpp/pull/13870 ? LLAMA_ROPE_TYPE_NEOX - : hparams.rope_type; - ggml_tensor * tmp; + : rp.type; - if (ggml_is_quantized(cur->type)) { - // dequantize to f32 -> RoPE -> quantize back - tmp = ggml_cast(ctx, cur, GGML_TYPE_F32); + const int64_t n_head_kv = hparams.n_head_kv(il); + const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(il); + const int64_t n_embd_head_k = hparams.n_embd_head_k(il); + const int64_t n_cells = ggml_nelements(k)/n_embd_k_gqa; - // rotate back - tmp = llama_mul_mat_hadamard(ctx, tmp, rot); + if (!ggml_is_quantized(k->type) && !rot && !rp.k_norm) { + // we rotate only the n_rot dimensions + ggml_tensor * k_rope = + ggml_view_3d(ctx, k, + rp.n_rot, n_head_kv, n_cells, + ggml_row_size(k->type, n_embd_head_k), + ggml_row_size(k->type, n_embd_k_gqa), + ggml_row_size(k->type, rp.n_offs)); - tmp = ggml_rope_ext(ctx, tmp, - shift, factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - yarn_ext_factor, yarn_attn_factor, yarn_beta_fast, yarn_beta_slow); - - // rotate fwd - tmp = llama_mul_mat_hadamard(ctx, tmp, rot); - - tmp = ggml_cpy(ctx, tmp, cur); - } else { - // we rotate only the first n_rot dimensions - tmp = ggml_rope_ext_inplace(ctx, cur, - shift, factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - yarn_ext_factor, yarn_attn_factor, yarn_beta_fast, yarn_beta_slow); + return ggml_rope_ext_inplace(ctx, k_rope, + shift, rp.factors, rp.n_rot, rope_type, rp.n_ctx_orig, rp.freq_base, rp.freq_scale, + rp.ext_factor, rp.attn_factor, rp.beta_fast, rp.beta_slow); } - return tmp; + // convert to f32 -> RoPE -> convert back, on whole rows as a head is not always a multiple of the block size + ggml_tensor * tmp = ggml_reshape_3d(ctx, ggml_cast(ctx, k, GGML_TYPE_F32), n_embd_head_k, n_head_kv, n_cells); + + // rotate back + if (rot) { + tmp = llama_mul_mat_hadamard(ctx, tmp, rot); + } + + // the RMS norm commutes with RoPE, but the weight does not + if (rp.k_norm) { + tmp = ggml_div(ctx, tmp, rp.k_norm); + } + + tmp = ggml_rope_ext(ctx, tmp, + shift, rp.factors, rp.n_rot, rope_type, rp.n_ctx_orig, rp.freq_base, rp.freq_scale, + rp.ext_factor, rp.attn_factor, rp.beta_fast, rp.beta_slow); + if (rp.n_offs > 0) { + tmp = ggml_rope_set_offset(tmp, rp.n_offs); + } + + if (rp.k_norm) { + tmp = ggml_mul(ctx, tmp, rp.k_norm); + } + + // rotate fwd + if (rot) { + tmp = llama_mul_mat_hadamard(ctx, tmp, rot); + } + + return ggml_cpy(ctx, tmp, k); } class llm_graph_input_k_shift : public llm_graph_input_i { @@ -2179,26 +2196,7 @@ ggml_cgraph * llama_kv_cache::build_graph_shift(llm_graph_result * res, llama_co continue; } - const int64_t n_head_kv = hparams.n_head_kv(il); - const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(il); - - const auto n_rot = hparams.n_rot(il); - const auto n_embd_head_k = hparams.n_embd_head_k(il); - const auto n_embd_nope = hparams.n_lora_kv > 0 ? n_embd_head_k - n_rot : 0; - - const float freq_base_l = model.get_rope_freq_base (cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - - ggml_tensor * k = - ggml_view_3d(ctx, layer.k, - n_rot, n_head_kv, get_size()*n_stream, - ggml_row_size(layer.k->type, n_embd_head_k), - ggml_row_size(layer.k->type, n_embd_k_gqa), - ggml_row_size(layer.k->type, n_embd_nope)); - - ggml_tensor * cur = build_rope_shift(cparams, ctx, k, inp->k_shift, layer.k_rot, rope_factors, freq_base_l, freq_scale_l, il); + ggml_tensor * cur = build_rope_shift(cparams, ctx, layer.k, inp->k_shift, layer.k_rot, il); ggml_build_forward_expand(gf, cur); } diff --git a/src/llama-kv-cache.h b/src/llama-kv-cache.h index 9ed36c562..5aafbebb3 100644 --- a/src/llama-kv-cache.h +++ b/src/llama-kv-cache.h @@ -114,7 +114,9 @@ public: const layer_reuse_cb & reuse, const layer_share_cb & share, // a model can hold more than one cache, so the tensor names have to stay unique - const char * name_tag = ""); + const char * name_tag = "", + // lightning indexer keys are rotated with the indexer RoPE parameters + bool is_indexer = false); ~llama_kv_cache() = default; @@ -283,6 +285,8 @@ private: // SWA const uint32_t n_swa = 0; + const bool is_indexer = false; + // env: LLAMA_ATTN_ROT_DISABLE uint32_t n_rot_k = 0; uint32_t n_rot_v = 0; @@ -339,12 +343,9 @@ private: ggml_tensor * build_rope_shift( const llama_cparams & cparams, ggml_context * ctx, - ggml_tensor * cur, + ggml_tensor * k, ggml_tensor * shift, ggml_tensor * rot, - ggml_tensor * factors, - float freq_base, - float freq_scale, uint32_t il) const; ggml_cgraph * build_graph_shift( diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 8c3ef97f7..0b063e845 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2856,6 +2856,36 @@ const ggml_tensor * llama_model::get_tensor(const char * name) const { return it->second; } +llama_rope_params llama_model::get_rope_params(const llama_cparams & cparams, int il) const { + llama_rope_params res; + + res.n_rot = hparams.has_rope(il) ? hparams.n_rot(il) : 0; + res.n_offs = hparams.n_lora_kv > 0 ? hparams.n_embd_head_k(il) - hparams.n_rot(il) : 0; + res.type = hparams.rope_type; + res.n_ctx_orig = cparams.n_ctx_orig_yarn; + res.freq_base = get_rope_freq_base (cparams, il); + res.freq_scale = get_rope_freq_scale(cparams, il); + res.ext_factor = cparams.yarn_ext_factor; + res.attn_factor = cparams.yarn_attn_factor; + res.beta_fast = cparams.yarn_beta_fast; + res.beta_slow = cparams.yarn_beta_slow; + res.factors = get_rope_factors(cparams, il); + + return res; +} + +llama_rope_params llama_model::get_rope_params_indexer(const llama_cparams & cparams, int il) const { + llama_rope_params res = get_rope_params(cparams, il); + + res.n_offs = 0; + res.type = LLAMA_ROPE_TYPE_NEOX; + res.freq_base = cparams.rope_freq_base; + res.freq_scale = cparams.rope_freq_scale; + res.factors = nullptr; + + return res; +} + float llama_model::get_rope_freq_base (const llama_cparams & cparams, int il) const { return hparams.is_swa(il) ? hparams.rope_freq_base_train_swa : cparams.rope_freq_base; } @@ -3631,6 +3661,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_KIMI_LINEAR: case LLM_ARCH_KIMI_K3: case LLM_ARCH_GLM5_NEXT: + case LLM_ARCH_STARCODER: return LLAMA_ROPE_TYPE_NONE; // use what we call a normal RoPE, operating on pairs of consecutive head values @@ -3639,7 +3670,6 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_LLAMA4: case LLM_ARCH_DECI: case LLM_ARCH_BAICHUAN: - case LLM_ARCH_STARCODER: case LLM_ARCH_INTERNLM2: case LLM_ARCH_MINICPM: case LLM_ARCH_XVERSE: diff --git a/src/llama-model.h b/src/llama-model.h index f8cffb346..9c0a51f3d 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -746,6 +746,23 @@ struct llama_hadamard { llama_hadamard_rotations inv; // latent lookup table -> inverse transform }; +struct llama_rope_params { + int32_t n_rot = 0; // 0 if the layer has no RoPE + int32_t n_offs = 0; // first rotated dim in a cached K head + int32_t type = LLAMA_ROPE_TYPE_NONE; + int32_t n_ctx_orig = 0; + + float freq_base = 0.0f; + float freq_scale = 0.0f; + float ext_factor = 0.0f; + float attn_factor = 0.0f; + float beta_fast = 0.0f; + float beta_slow = 0.0f; + + ggml_tensor * factors = nullptr; + ggml_tensor * k_norm = nullptr; // K norm weight applied after RoPE, a K-shift has to undo it +}; + struct llama_model { llm_type type = LLM_TYPE_UNKNOWN; llm_arch arch = LLM_ARCH_UNKNOWN; @@ -971,6 +988,12 @@ struct llama_model { const struct ggml_tensor * get_tensor(const char * name) const; + // the RoPE of a layer, used by both the model graph and the K-shift + virtual llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const; + + // same, for the keys of the lightning indexer + virtual llama_rope_params get_rope_params_indexer(const llama_cparams & cparams, int il) const; + float get_rope_freq_base (const llama_cparams & cparams, int il) const; float get_rope_freq_scale(const llama_cparams & cparams, int il) const; diff --git a/src/models/afmoe.cpp b/src/models/afmoe.cpp index 75da5e2eb..0eb208f63 100644 --- a/src/models/afmoe.cpp +++ b/src/models/afmoe.cpp @@ -23,6 +23,10 @@ void llama_model_afmoe::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_NONE; } + for (uint32_t il = 0; il < hparams.n_layer_all; ++il) { + hparams.rope_pattern[il] = (il + 1) % hparams.n_no_rope_layer_step != 0; + } + // Default to sigmoid if not set if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) { hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID; @@ -126,15 +130,9 @@ llama_model_afmoe::graph::graph(const llm_graph_params & params) : llm_graph_con const float kq_scale = 1.0f/sqrtf(float(n_embd_head)); for (int il = 0; il < n_layer; ++il) { - const float freq_base_l = model.get_rope_freq_base (cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); ggml_tensor * inpSA = inpL; - // This overlaps with SWA layers in current models, so get_rope_freq_base/scale may be superfluous - const bool use_rope = hparams.n_no_rope_layer_step > 0 && - (il + 1) % hparams.n_no_rope_layer_step != 0; - // dual attention normalization (pre) cur = build_norm(inpL, model.layers[il].attn_norm, NULL, @@ -158,19 +156,11 @@ llama_model_afmoe::graph::graph(const llm_graph_params & params) : llm_graph_con cb(Qcur, "Qcur_normed", il); cb(Kcur, "Kcur_normed", il); - if (use_rope) { - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Qcur, "Qcur_rope", il); + Qcur = build_rope(Qcur, inp_pos, il); + cb(Qcur, "Qcur_rope", il); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Kcur, "Kcur_rope", il); - } + Kcur = build_rope(Kcur, inp_pos, il); + cb(Kcur, "Kcur_rope", il); cur = build_attn(inp_attn, NULL, NULL, NULL, // wo will be applied after gating diff --git a/src/models/apertus.cpp b/src/models/apertus.cpp index 2b1ebc965..c0bb23659 100644 --- a/src/models/apertus.cpp +++ b/src/models/apertus.cpp @@ -84,8 +84,6 @@ llama_model_apertus::graph::graph(const llm_graph_params & params) : llm_graph_c // self-attention { - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); @@ -96,11 +94,8 @@ llama_model_apertus::graph::graph(const llm_graph_params & params) : llm_graph_c Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur_pos", il); cb(Kcur, "Kcur_pos", il); diff --git a/src/models/arcee.cpp b/src/models/arcee.cpp index c58025ccd..58607d82e 100644 --- a/src/models/arcee.cpp +++ b/src/models/arcee.cpp @@ -76,24 +76,12 @@ llama_model_arcee::graph::graph(const llm_graph_params & params) : llm_graph_con // self-attention { - // rope freq factors for llama3; may return nullptr for llama2 and other models - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/arctic.cpp b/src/models/arctic.cpp index e9db89d99..a72f2043b 100644 --- a/src/models/arctic.cpp +++ b/src/models/arctic.cpp @@ -86,17 +86,8 @@ llama_model_arctic::graph::graph(const llm_graph_params & params) : llm_graph_co auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/baichuan.cpp b/src/models/baichuan.cpp index e7315b304..3bccc0bff 100644 --- a/src/models/baichuan.cpp +++ b/src/models/baichuan.cpp @@ -12,6 +12,11 @@ void llama_model_baichuan::load_arch_hparams(llama_model_loader & ml) { // TODO: become GGUF KV parameter hparams.f_max_alibi_bias = 8.0f; } + + // only the 7B uses RoPE + if (type != LLM_TYPE_7B) { + std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), 0); + } } void llama_model_baichuan::load_arch_tensors(llama_model_loader &) { @@ -74,25 +79,8 @@ llama_model_baichuan::graph::graph(const llm_graph_params & params) : llm_graph_ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - switch (model.type) { - case LLM_TYPE_7B: - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - break; - case LLM_TYPE_13B: - case LLM_TYPE_UNKNOWN: - break; - default: - GGML_ABORT("fatal error"); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/bailingmoe.cpp b/src/models/bailingmoe.cpp index b8e1791fc..890dcaa7b 100644 --- a/src/models/bailingmoe.cpp +++ b/src/models/bailingmoe.cpp @@ -83,24 +83,12 @@ llama_model_bailingmoe::graph::graph(const llm_graph_params & params) : llm_grap // self-attention { - // rope freq factors for llama3; may return nullptr for llama2 and other models - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head_k, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/bailingmoe2.cpp b/src/models/bailingmoe2.cpp index 05f08c84f..651623e0e 100644 --- a/src/models/bailingmoe2.cpp +++ b/src/models/bailingmoe2.cpp @@ -120,14 +120,12 @@ llama_model_bailingmoe2::graph::graph(const llm_graph_params & params) : Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/bailingmoe3.cpp b/src/models/bailingmoe3.cpp index a904dca8a..8e47e2a0a 100644 --- a/src/models/bailingmoe3.cpp +++ b/src/models/bailingmoe3.cpp @@ -205,9 +205,6 @@ llama_model_bailingmoe3::graph::graph(const llm_graph_params & params) : const int64_t n_seqs = ubatch.n_seqs; const int64_t n_seq_tokens = ubatch.n_seq_tokens; - const bool use_mrope = hparams.use_mrope(); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); const int64_t qk_head_dim = hparams.n_embd_head_k_mla(); const int64_t v_head_dim = hparams.n_embd_head_v_mla(); const int64_t qk_rope_head_dim = hparams.n_rot(); @@ -299,17 +296,8 @@ llama_model_bailingmoe3::graph::graph(const llm_graph_params & params) : ggml_row_size(kv_all->type, kv_lora_rank + qk_rope_head_dim), ggml_row_size(kv_all->type, kv_lora_rank)); - if (use_mrope) { - q_pe = ggml_rope_multi(ctx0, q_pe, inp_pos, nullptr, n_rot, sections, rope_type, - n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - k_pe = ggml_rope_multi(ctx0, k_pe, inp_pos, nullptr, n_rot, sections, rope_type, - n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - } else { - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - } + q_pe = build_rope(q_pe, inp_pos, il); + k_pe = build_rope(k_pe, inp_pos, il); kv = build_norm(kv, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3); @@ -463,21 +451,8 @@ llama_model_bailingmoe3::graph_mtp::graph_mtp(const llm_graph_params & params) : ggml_row_size(kv_all->type, kv_lora_rank + qk_rope_head_dim), ggml_row_size(kv_all->type, kv_lora_rank)); - const bool use_mrope = hparams.use_mrope(); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - - if (use_mrope) { - q_pe = ggml_rope_multi(ctx0, q_pe, inp_pos, nullptr, n_rot, sections, rope_type, - n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - k_pe = ggml_rope_multi(ctx0, k_pe, inp_pos, nullptr, n_rot, sections, rope_type, - n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - } else { - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - } + q_pe = build_rope(q_pe, inp_pos, il); + k_pe = build_rope(k_pe, inp_pos, il); kv = build_norm(kv, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3); diff --git a/src/models/beam.cpp b/src/models/beam.cpp index ec4fc1436..8654e0c95 100644 --- a/src/models/beam.cpp +++ b/src/models/beam.cpp @@ -156,15 +156,9 @@ llama_model_beam::graph::graph(const llm_graph_params & params) : llm_graph_cont cb(Kcur, "Kcur_normed", il); if (is_swa) { - const float freq_base_l = model.get_rope_freq_base (cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur_rope", il); cb(Kcur, "Kcur_rope", il); } else { diff --git a/src/models/bert.cpp b/src/models/bert.cpp index 2b2108dc4..b98faffc9 100644 --- a/src/models/bert.cpp +++ b/src/models/bert.cpp @@ -131,11 +131,8 @@ llama_model_bert::graph::graph(const llm_graph_params & params) : llm_graph_cont // RoPE if (model.arch == LLM_ARCH_NOMIC_BERT || model.arch == LLM_ARCH_NOMIC_BERT_MOE || model.arch == LLM_ARCH_JINA_BERT_V3) { - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); } cb(Qcur, "Qcur", il); diff --git a/src/models/bitnet.cpp b/src/models/bitnet.cpp index 3c366e13f..caee27a3d 100644 --- a/src/models/bitnet.cpp +++ b/src/models/bitnet.cpp @@ -78,17 +78,8 @@ llama_model_bitnet::graph::graph(const llm_graph_params & params) : llm_graph_co auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/chameleon.cpp b/src/models/chameleon.cpp index 496538506..ab16927c7 100644 --- a/src/models/chameleon.cpp +++ b/src/models/chameleon.cpp @@ -103,17 +103,8 @@ llama_model_chameleon::graph::graph(const llm_graph_params & params) : llm_graph cb(Kcur, "Kcur", il); } - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/chatglm.cpp b/src/models/chatglm.cpp index 33c6386b6..e7442b75f 100644 --- a/src/models/chatglm.cpp +++ b/src/models/chatglm.cpp @@ -87,17 +87,8 @@ llama_model_chatglm::graph::graph(const llm_graph_params & params) : llm_graph_c n_embd_head, n_head, n_head_kv, il); //printf("freq_base: %f freq_scale: %f ext_factor: %f attn_factor: %f\n", freq_base, freq_scale, ext_factor, attn_factor); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/clef.cpp b/src/models/clef.cpp index 03d3e595d..3ac5782a8 100644 --- a/src/models/clef.cpp +++ b/src/models/clef.cpp @@ -249,9 +249,6 @@ llama_model_clef::graph::graph(const llm_graph_params & params) : GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - ggml_tensor * cur; ggml_tensor * inpL; @@ -274,7 +271,7 @@ llama_model_clef::graph::graph(const llm_graph_params & params) : if (hparams.is_recr(il)) { cur = build_layer_attn_linear(cur, il); } else { - cur = build_layer_attn(inp_attn, cur, inp_pos, sections, il); + cur = build_layer_attn(inp_attn, cur, inp_pos, il); } cur = ggml_add(ctx0, cur, inpSA); @@ -359,7 +356,6 @@ ggml_tensor * llama_model_clef::graph::build_layer_attn( llm_graph_input_attn_no_cache * inp, ggml_tensor * cur, ggml_tensor * inp_pos, - int * sections, int il) { const int64_t n_embd_head = hparams.n_embd_head_v(); @@ -389,17 +385,8 @@ ggml_tensor * llama_model_clef::graph::build_layer_attn( Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); - Qcur = ggml_rope_multi( - ctx0, Qcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_multi( - ctx0, Kcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale; diff --git a/src/models/clover-moe.cpp b/src/models/clover-moe.cpp index 498ca275b..9caaa5630 100644 --- a/src/models/clover-moe.cpp +++ b/src/models/clover-moe.cpp @@ -252,13 +252,8 @@ ggml_tensor * llama_model_clover_moe::graph::build_layer_attn( Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/codeshell.cpp b/src/models/codeshell.cpp index 1d499684c..e316602f5 100644 --- a/src/models/codeshell.cpp +++ b/src/models/codeshell.cpp @@ -80,17 +80,8 @@ llama_model_codeshell::graph::graph(const llm_graph_params & params) : llm_graph auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/cogvlm.cpp b/src/models/cogvlm.cpp index c878a3bf0..657b7d89d 100644 --- a/src/models/cogvlm.cpp +++ b/src/models/cogvlm.cpp @@ -46,6 +46,22 @@ void llama_model_cogvlm::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_cogvlm::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + // plain ggml_rope + res.n_ctx_orig = 0; + res.freq_base = 10000.0f; + res.freq_scale = 1.0f; + res.ext_factor = 0.0f; + res.attn_factor = 1.0f; + res.beta_fast = 0.0f; + res.beta_slow = 0.0f; + res.factors = nullptr; + + return res; +} + std::unique_ptr llama_model_cogvlm::build_arch_graph(const llm_graph_params & params) const { return std::make_unique(params); } @@ -113,8 +129,8 @@ llama_model_cogvlm::graph::graph(const llm_graph_params & params) : ggml_tensor * Vcur = ggml_view_3d(ctx0, qkv, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float), qkv->nb[1], 2 * n_embd * ggml_element_size(qkv)); - Qcur = ggml_rope(ctx0, Qcur, inp_pos, n_embd_head, rope_type); - Kcur = ggml_rope(ctx0, Kcur, inp_pos, n_embd_head, rope_type); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cur = build_attn(inp_attn, wo, nullptr, wo_s, diff --git a/src/models/cohere2.cpp b/src/models/cohere2.cpp index acde0fea6..251b2da62 100644 --- a/src/models/cohere2.cpp +++ b/src/models/cohere2.cpp @@ -3,6 +3,7 @@ void llama_model_cohere2::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; load_swa_pattern(ml, 4); + hparams.rope_pattern = hparams.is_swa_impl; hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; @@ -43,6 +44,16 @@ void llama_model_cohere2::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_cohere2::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + // the SWA layers use the global base and scale + res.freq_base = cparams.rope_freq_base; + res.freq_scale = cparams.rope_freq_scale; + + return res; +} + std::unique_ptr llama_model_cohere2::build_arch_graph(const llm_graph_params & params) const { return std::make_unique(params); } @@ -67,10 +78,7 @@ llama_model_cohere2::graph::graph(const llm_graph_params & params) : llm_graph_c ggml_tensor * inp_out_ids = build_inp_out_ids(); for (int il = 0; il < n_layer; ++il) { - const bool is_swa = hparams.is_swa(il); // UNUSED: - // const float freq_base_l = model.get_rope_freq_base (cparams, il); - // const float freq_scale_l = model.get_rope_freq_scale(cparams, il); // norm cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM, il); @@ -79,26 +87,12 @@ llama_model_cohere2::graph::graph(const llm_graph_params & params) : llm_graph_c // self-attention { - // rope freq factors for 128k context - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - if (is_swa) { - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/cohere2moe.cpp b/src/models/cohere2moe.cpp index f673c99bc..5bb2db15a 100644 --- a/src/models/cohere2moe.cpp +++ b/src/models/cohere2moe.cpp @@ -27,6 +27,11 @@ void llama_model_cohere2moe::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; load_swa_pattern(ml, 4, true); + // the dense prefix and the MTP layers use RoPE even if they are not SWA + for (uint32_t il = 0; il < hparams.n_layer_all; ++il) { + hparams.rope_pattern[il] = hparams.is_swa(il) || il < hparams.n_layer_dense_lead || il >= hparams.n_layer(); + } + hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); @@ -133,6 +138,16 @@ void llama_model_cohere2moe::load_arch_tensors(llama_model_loader & ml) { } } +llama_rope_params llama_model_cohere2moe::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + // the SWA layers use the global base and scale + res.freq_base = cparams.rope_freq_base; + res.freq_scale = cparams.rope_freq_scale; + + return res; +} + std::unique_ptr llama_model_cohere2moe::build_arch_graph(const llm_graph_params & params) const { if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { return std::make_unique(params); @@ -157,9 +172,6 @@ llama_model_cohere2moe::graph::graph(const llm_graph_params & params) : llm_grap // MTP/NextN layers are loaded as extra decoder blocks but not executed in the main pass. for (int il = 0; il < n_layer; ++il) { - const bool is_swa = hparams.is_swa(il); - // Dense-prefix full-attention layers use RoPE; later layers follow the SWA pattern. - const bool force_rope = static_cast(il) < hparams.n_layer_dense_lead; cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, cohere2moe_norm_type, il); cb(cur, "attn_norm", il); @@ -172,19 +184,8 @@ llama_model_cohere2moe::graph::graph(const llm_graph_params & params) : llm_grap auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head, n_head, n_head_kv, il); - if (is_swa || force_rope) { - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); @@ -346,15 +347,8 @@ llama_model_cohere2moe::graph_mtp::graph_mtp(const llm_graph_params & params) : ggml_tensor * ffn_inp = cur; auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head, n_head, n_head_kv, il); - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "mtp_Qcur", il); cb(Kcur, "mtp_Kcur", il); diff --git a/src/models/command-r.cpp b/src/models/command-r.cpp index dd1505186..bea97e6d8 100644 --- a/src/models/command-r.cpp +++ b/src/models/command-r.cpp @@ -80,15 +80,13 @@ llama_model_command_r::graph::graph(const llm_graph_params & params) : Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM, il); cb(Qcur, "Qcur", il); } - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); if (model.layers[il].attn_k_norm) { Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM, il); cb(Kcur, "Kcur", il); } - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/dbrx.cpp b/src/models/dbrx.cpp index 5b4c87d0c..343040e0b 100644 --- a/src/models/dbrx.cpp +++ b/src/models/dbrx.cpp @@ -76,17 +76,8 @@ llama_model_dbrx::graph::graph(const llm_graph_params & params) : llm_graph_cont auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/deci.cpp b/src/models/deci.cpp index 9a5beecc1..114b12640 100644 --- a/src/models/deci.cpp +++ b/src/models/deci.cpp @@ -118,18 +118,13 @@ llama_model_deci::graph::graph(const llm_graph_params & params) : llm_graph_cont cb(cur, "wo", il); } else if (n_head > 0) { // self-attention - // rope freq factors for llama3; may return nullptr for llama2 and other models - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/deepseek.cpp b/src/models/deepseek.cpp index 76760a560..c6a6d6a97 100644 --- a/src/models/deepseek.cpp +++ b/src/models/deepseek.cpp @@ -102,18 +102,12 @@ llama_model_deepseek::graph::graph(const llm_graph_params & params) : // self-attention { - // rope freq factors for llama3; may return nullptr for llama2 and other models - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/deepseek2.cpp b/src/models/deepseek2.cpp index 36c8ea93c..7ed35ee9a 100644 --- a/src/models/deepseek2.cpp +++ b/src/models/deepseek2.cpp @@ -308,14 +308,10 @@ llama_model_deepseek2::graph_mtp::graph_mtp(const llm_graph_params & params) : const float kq_scale = 1.0f * mscale * mscale / sqrtf(float(n_embd_head_k_mla)); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + q_pe = build_rope(q_pe, inp_pos, il); cb(q_pe, "mtp_q_pe_rope", il); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "mtp_k_pe_rope", il); q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3); @@ -466,7 +462,6 @@ llama_model_deepseek2::graph::graph(const llm_graph_params & params) : // self_attention if (is_ocr) { const int n_embed_head = hparams.n_embd / hparams.n_head(); - const int ocr_rope_type = GGML_ROPE_TYPE_NEOX; GGML_ASSERT(n_embed_head == n_embd_head_k && n_embed_head == n_embd_head_v); auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, @@ -476,8 +471,8 @@ llama_model_deepseek2::graph::graph(const llm_graph_params & params) : cb(Vcur, "v", il); GGML_ASSERT(fabs(freq_base - 10000.0) < 1e-4); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_embed_head, ocr_rope_type, 0, freq_base, 1, 0, 1, 0, 0); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_embed_head, ocr_rope_type, 0, freq_base, 1, 0, 1, 0, 0); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "q_pe", il); cb(Kcur, "k_pe", il); @@ -524,8 +519,7 @@ llama_model_deepseek2::graph::graph(const llm_graph_params & params) : ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); cb(k_pe, "k_pe", il); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "k_pe", il); kv_cmpr = build_norm(kv_cmpr, model.layers[il].attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); @@ -542,8 +536,7 @@ llama_model_deepseek2::graph::graph(const llm_graph_params & params) : q->nb[1], q->nb[2], ggml_row_size(q->type, n_embd_head_qk_nope)); cb(q_pe, "q_pe", il); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + q_pe = build_rope(q_pe, inp_pos, il); cb(q_pe, "q_pe", il); // {n_embd_head_qk_nope, n_tokens, n_head} @@ -606,8 +599,7 @@ llama_model_deepseek2::graph::graph(const llm_graph_params & params) : cb(Vcur, "Vcur_cont", il); // RoPE is applied to the trailing dims only - ggml_tensor * Qcur = ggml_rope_ext(ctx0, q, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + ggml_tensor * Qcur = build_rope(q, inp_pos, il); Qcur = ggml_rope_set_offset(Qcur, n_embd_head_qk_nope); cb(Qcur, "Qcur", il); diff --git a/src/models/deepseek2ocr.cpp b/src/models/deepseek2ocr.cpp index 88026c3f8..3357f1b13 100644 --- a/src/models/deepseek2ocr.cpp +++ b/src/models/deepseek2ocr.cpp @@ -74,6 +74,23 @@ void llama_model_deepseek2ocr::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_deepseek2ocr::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + res.n_rot = hparams.n_embd_head_k(il); + res.n_offs = 0; + res.type = LLAMA_ROPE_TYPE_NEOX; + res.n_ctx_orig = 0; + res.freq_scale = 1.0f; + res.ext_factor = 0.0f; + res.attn_factor = 1.0f; + res.beta_fast = 0.0f; + res.beta_slow = 0.0f; + res.factors = nullptr; + + return res; +} + std::unique_ptr llama_model_deepseek2ocr::build_arch_graph(const llm_graph_params & params) const { return std::make_unique(params); } diff --git a/src/models/deepseek32.cpp b/src/models/deepseek32.cpp index d208c524d..6a7a0a7a9 100644 --- a/src/models/deepseek32.cpp +++ b/src/models/deepseek32.cpp @@ -224,9 +224,7 @@ llama_model_deepseek32::graph::graph(const llm_graph_params & params) : // {n_embd_indexer_head, n_indexer_head, n_tokens} indexer_q = ggml_reshape_3d(ctx0, indexer_q, n_embd_indexer_head, n_indexer_head, n_tokens); - indexer_q = ggml_rope_ext(ctx0, indexer_q, inp_pos, nullptr, n_rot, - LLAMA_ROPE_TYPE_NEOX, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + indexer_q = build_rope_indexer(indexer_q, inp_pos, il); cb(indexer_q, "indexer_q", il); ggml_tensor * indexer_k = ggml_mul_mat(ctx0, model.layers[il].indexer_attn_k, cur); @@ -237,9 +235,7 @@ llama_model_deepseek32::graph::graph(const llm_graph_params & params) : // {n_embd_indexer_head, 1, n_tokens} indexer_k = ggml_reshape_3d(ctx0, indexer_k, n_embd_indexer_head, 1, n_tokens); - indexer_k = ggml_rope_ext(ctx0, indexer_k, inp_pos, nullptr, n_rot, - LLAMA_ROPE_TYPE_NEOX, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + indexer_k = build_rope_indexer(indexer_k, inp_pos, il); cb(indexer_k, "indexer_k", il); // perform Hadamard transform on indexer q and k @@ -347,12 +343,10 @@ llama_model_deepseek32::graph::graph(const llm_graph_params & params) : ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); cb(k_pe, "k_pe", il); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + q_pe = build_rope(q_pe, inp_pos, il); cb(q_pe, "q_pe", il); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "k_pe", il); kv_cmpr = build_norm(kv_cmpr, model.layers[il].attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); @@ -607,12 +601,10 @@ llama_model_deepseek32::graph_mtp::graph_mtp(const llm_graph_params & params) ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); cb(k_pe, "mtp_k_pe", il); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + q_pe = build_rope(q_pe, inp_pos, il); cb(q_pe, "mtp_q_pe", il); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "mtp_k_pe", il); kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); diff --git a/src/models/dots1.cpp b/src/models/dots1.cpp index 41ffd7101..c2fa96560 100644 --- a/src/models/dots1.cpp +++ b/src/models/dots1.cpp @@ -106,14 +106,12 @@ llama_model_dots1::graph::graph(const llm_graph_params & params) : Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/dots3note.cpp b/src/models/dots3note.cpp index c05337494..a6fccd58f 100644 --- a/src/models/dots3note.cpp +++ b/src/models/dots3note.cpp @@ -145,6 +145,14 @@ void llama_model_dots3note::load_arch_tensors(llama_model_loader & ml) { } } +llama_rope_params llama_model_dots3note::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + res.freq_scale = cparams.rope_freq_scale; + + return res; +} + std::unique_ptr llama_model_dots3note::build_arch_graph(const llm_graph_params & params) const { return std::make_unique(params); } @@ -186,7 +194,6 @@ llama_model_dots3note::graph::graph(const llm_graph_params & params) : const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope; const float kq_scale = 1.0f/sqrtf(float(n_embd_head_k_mla)); - const float freq_base_l = model.get_rope_freq_base(cparams, il); // norm cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); @@ -211,9 +218,7 @@ llama_model_dots3note::graph::graph(const llm_graph_params & params) : // {n_embd_indexer_head, n_indexer_head, n_tokens} indexer_q = ggml_reshape_3d(ctx0, indexer_q, n_embd_indexer_head, n_indexer_head, n_tokens); - indexer_q = ggml_rope_ext(ctx0, indexer_q, inp_pos, nullptr, n_rot, - LLAMA_ROPE_TYPE_NEOX, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + indexer_q = build_rope_indexer(indexer_q, inp_pos, il); cb(indexer_q, "indexer_q", il); ggml_tensor * indexer_k = ggml_mul_mat(ctx0, model.layers[il].indexer_attn_k, cur); @@ -224,9 +229,7 @@ llama_model_dots3note::graph::graph(const llm_graph_params & params) : // {n_embd_indexer_head, 1, n_tokens} indexer_k = ggml_reshape_3d(ctx0, indexer_k, n_embd_indexer_head, 1, n_tokens); - indexer_k = ggml_rope_ext(ctx0, indexer_k, inp_pos, nullptr, n_rot, - LLAMA_ROPE_TYPE_NEOX, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + indexer_k = build_rope_indexer(indexer_k, inp_pos, il); cb(indexer_k, "indexer_k", il); // perform Hadamard transform on indexer q and k @@ -332,12 +335,10 @@ llama_model_dots3note::graph::graph(const llm_graph_params & params) : k_pe = build_norm(k_pe, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il); cb(k_pe, "k_pe", il); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + q_pe = build_rope(q_pe, inp_pos, il); cb(q_pe, "q_pe", il); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "k_pe", il); kv_cmpr = build_norm(kv_cmpr, model.layers[il].attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); diff --git a/src/models/dream.cpp b/src/models/dream.cpp index 173794f57..76c0b03c8 100644 --- a/src/models/dream.cpp +++ b/src/models/dream.cpp @@ -81,11 +81,8 @@ llama_model_dream::graph::graph(const llm_graph_params & params) : auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/eagle3.cpp b/src/models/eagle3.cpp index 580f44e59..990b9fe5d 100644 --- a/src/models/eagle3.cpp +++ b/src/models/eagle3.cpp @@ -234,20 +234,9 @@ llama_model_eagle3::graph::graph(const llm_graph_params & params) : llm_g Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); - // rope freq factors, returns nullptr if not available - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // RoPE - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur_rope", il); cb(Kcur, "Kcur_rope", il); diff --git a/src/models/ernie4-5-moe.cpp b/src/models/ernie4-5-moe.cpp index e5ebe3955..7edf9f508 100644 --- a/src/models/ernie4-5-moe.cpp +++ b/src/models/ernie4-5-moe.cpp @@ -37,11 +37,8 @@ llama_model_ernie4_5_moe::graph::graph(const llm_graph_params & params) : auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/ernie4-5.cpp b/src/models/ernie4-5.cpp index 28197fb0a..932cc74ce 100644 --- a/src/models/ernie4-5.cpp +++ b/src/models/ernie4-5.cpp @@ -105,11 +105,8 @@ llama_model_ernie4_5::graph::graph(const llm_graph_params & params) : auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/eurobert.cpp b/src/models/eurobert.cpp index 3e0d0902d..7a42f0dbf 100644 --- a/src/models/eurobert.cpp +++ b/src/models/eurobert.cpp @@ -62,17 +62,8 @@ llama_model_eurobert::graph::graph(const llm_graph_params & params) : llm_graph_ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/exaone-moe.cpp b/src/models/exaone-moe.cpp index c7cd77da2..15bbd5a3f 100644 --- a/src/models/exaone-moe.cpp +++ b/src/models/exaone-moe.cpp @@ -4,6 +4,7 @@ void llama_model_exaone_moe::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; hparams.n_swa = 128; load_swa_pattern(ml, 4); + hparams.rope_pattern = hparams.is_swa_impl; hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; @@ -101,6 +102,16 @@ void llama_model_exaone_moe::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_exaone_moe::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + // the SWA layers use the global base and scale + res.freq_base = cparams.rope_freq_base; + res.freq_scale = cparams.rope_freq_scale; + + return res; +} + std::unique_ptr llama_model_exaone_moe::build_arch_graph(const llm_graph_params & params) const { return std::make_unique(params); } @@ -127,17 +138,12 @@ llama_model_exaone_moe::graph::graph(const llm_graph_params & params) : for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; - // use RoPE for SWA layers - const bool is_local_layer = hparams.is_swa(il); - // norm cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); // self-attention { - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); @@ -147,13 +153,8 @@ llama_model_exaone_moe::graph::graph(const llm_graph_params & params) : cb(Qcur, "Qcur_normed", il); cb(Kcur, "Kcur_normed", il); - if (is_local_layer) { - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, - freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, - freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); diff --git a/src/models/exaone.cpp b/src/models/exaone.cpp index cd6611aee..780f34f04 100644 --- a/src/models/exaone.cpp +++ b/src/models/exaone.cpp @@ -71,18 +71,12 @@ llama_model_exaone::graph::graph(const llm_graph_params & params) : // self-attention { - // rope freq factors for llama3; may return nullptr for llama2 and other models - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/exaone4.cpp b/src/models/exaone4.cpp index de141b173..bca08e061 100644 --- a/src/models/exaone4.cpp +++ b/src/models/exaone4.cpp @@ -5,6 +5,7 @@ void llama_model_exaone4::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; hparams.n_swa = 4096; load_swa_pattern(ml, 4); + hparams.rope_pattern = hparams.is_swa_impl; hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; @@ -70,6 +71,16 @@ void llama_model_exaone4::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_exaone4::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + // the SWA layers use the global base and scale + res.freq_base = cparams.rope_freq_base; + res.freq_scale = cparams.rope_freq_scale; + + return res; +} + std::unique_ptr llama_model_exaone4::build_arch_graph(const llm_graph_params & params) const { if (hparams.swa_type == LLAMA_SWA_TYPE_STANDARD) { return std::make_unique>(params); @@ -108,13 +119,11 @@ llama_model_exaone4::graph::graph(const llm_graph_params & params) : ggml_tensor * inpSA = inpL; // use RoPE for SWA layers or non-SWA models - const bool use_rope = hparams.is_swa(il) || hparams.swa_type == LLAMA_SWA_TYPE_NONE; cur = inpL; // self-attention { - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); @@ -124,13 +133,8 @@ llama_model_exaone4::graph::graph(const llm_graph_params & params) : cb(Qcur, "Qcur_normed", il); cb(Kcur, "Kcur_normed", il); - if (use_rope) { - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, - freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, - freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); diff --git a/src/models/falcon-h1.cpp b/src/models/falcon-h1.cpp index 08bd20240..22c677b47 100644 --- a/src/models/falcon-h1.cpp +++ b/src/models/falcon-h1.cpp @@ -105,6 +105,14 @@ void llama_model_falcon_h1::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_falcon_h1::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + res.factors = nullptr; + + return res; +} + std::unique_ptr llama_model_falcon_h1::build_arch_graph(const llm_graph_params & params) const { return std::make_unique(params); } @@ -139,11 +147,8 @@ llama_model_falcon_h1::graph::graph(const llm_graph_params & params) : auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, hparams.rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, hparams.rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur-post-rope", il); cb(Kcur, "Kcur-post-rope", il); diff --git a/src/models/falcon.cpp b/src/models/falcon.cpp index 5eaecfd8f..6269e633c 100644 --- a/src/models/falcon.cpp +++ b/src/models/falcon.cpp @@ -91,17 +91,8 @@ llama_model_falcon::graph::graph(const llm_graph_params & params) : llm_graph_co n_embd_head, n_head, n_head_kv, il); // using mode = 2 for neox mode - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/flint.cpp b/src/models/flint.cpp index a2995ff4d..520334ce6 100644 --- a/src/models/flint.cpp +++ b/src/models/flint.cpp @@ -70,6 +70,16 @@ void llama_model_flint::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_flint::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + // the SWA layers use the global base and scale + res.freq_base = cparams.rope_freq_base; + res.freq_scale = cparams.rope_freq_scale; + + return res; +} + std::unique_ptr llama_model_flint::build_arch_graph(const llm_graph_params & params) const { return std::make_unique(params); } @@ -109,12 +119,8 @@ llama_model_flint::graph::graph(const llm_graph_params & params) : Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il); - if (hparams.has_rope(il)) { - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); diff --git a/src/models/gemma-embedding.cpp b/src/models/gemma-embedding.cpp index 9c5dc9825..aa43910be 100644 --- a/src/models/gemma-embedding.cpp +++ b/src/models/gemma-embedding.cpp @@ -89,8 +89,6 @@ llama_model_gemma_embedding::graph::graph(const llm_graph_params & params) : ggml_tensor * inp_out_ids = build_inp_out_ids(); for (int il = 0; il < n_layer; ++il) { - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); // norm cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); @@ -105,14 +103,12 @@ llama_model_gemma_embedding::graph::graph(const llm_graph_params & params) : Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/gemma-embedding2.cpp b/src/models/gemma-embedding2.cpp index d81903b63..920827dc5 100644 --- a/src/models/gemma-embedding2.cpp +++ b/src/models/gemma-embedding2.cpp @@ -100,10 +100,6 @@ llama_model_gemma_embedding2::graph::graph(const llm_graph_params & params) : const int64_t n_head = hparams.n_head(il); const int64_t n_head_kv = hparams.n_head_kv(il); - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - const int n_rot_l = hparams.n_rot(il); - // norm cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); @@ -128,10 +124,8 @@ llama_model_gemma_embedding2::graph::graph(const llm_graph_params & params) : cb(Kcur, "Kcur_normed", il); cb(Vcur, "Vcur_normed", il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur_pos", il); cb(Kcur, "Kcur_pos", il); diff --git a/src/models/gemma.cpp b/src/models/gemma.cpp index 03f917923..1d42acd01 100644 --- a/src/models/gemma.cpp +++ b/src/models/gemma.cpp @@ -69,15 +69,8 @@ llama_model_gemma::graph::graph(const llm_graph_params & params) : llm_graph_con auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/gemma2.cpp b/src/models/gemma2.cpp index 0a6a6f42a..826396ce1 100644 --- a/src/models/gemma2.cpp +++ b/src/models/gemma2.cpp @@ -76,8 +76,6 @@ llama_model_gemma2::graph::graph(const llm_graph_params & params) : llm_graph_co ggml_tensor * inp_out_ids = build_inp_out_ids(); for (int il = 0; il < n_layer; ++il) { - const float freq_base_l = model.get_rope_freq_base (cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); // norm cur = build_norm(inpL, @@ -91,15 +89,8 @@ llama_model_gemma2::graph::graph(const llm_graph_params & params) : llm_graph_co auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/gemma3.cpp b/src/models/gemma3.cpp index 907d35fb1..9520c3fcf 100644 --- a/src/models/gemma3.cpp +++ b/src/models/gemma3.cpp @@ -105,17 +105,6 @@ llama_model_gemma3::graph::graph(const llm_graph_params & params) : llm_gr ggml_tensor * inp_out_ids = build_inp_out_ids(); for (int il = 0; il < n_layer; ++il) { - float freq_base_l = 0.0f; - float freq_scale_l = 0.0f; - - if constexpr (iswa) { - freq_base_l = model.get_rope_freq_base (cparams, il); - freq_scale_l = model.get_rope_freq_scale(cparams, il); - } else { - freq_base_l = freq_base; - freq_scale_l = freq_scale; - } - // norm cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); @@ -129,18 +118,12 @@ llama_model_gemma3::graph::graph(const llm_graph_params & params) : llm_gr Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/gemma3n.cpp b/src/models/gemma3n.cpp index 90c062aff..64809def8 100644 --- a/src/models/gemma3n.cpp +++ b/src/models/gemma3n.cpp @@ -132,8 +132,6 @@ llama_model_gemma3n::graph::graph(const llm_graph_params & params) : for (int il = 0; il < n_layer; ++il) { // this block is made to be closely resemble Gemma3p5DecoderLayer on python code - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); ggml_tensor * cur = inpL; // [n_embd, n_tokens, n_altup] ggml_tensor * predictions = altup_predict(cur, il); // [n_embd, n_tokens, n_altup] @@ -162,11 +160,8 @@ llama_model_gemma3n::graph::graph(const llm_graph_params & params) : cb(Kcur, "Kcur_normed", il); cb(Vcur, "Vcur_normed", il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur_pos", il); cb(Kcur, "Kcur_pos", il); @@ -190,8 +185,7 @@ llama_model_gemma3n::graph::graph(const llm_graph_params & params) : Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); cb(Qcur, "Qcur_pos", il); cur = build_attn(inp_attn, diff --git a/src/models/gemma4-assistant.cpp b/src/models/gemma4-assistant.cpp index b6e2d5af1..77968cbb3 100644 --- a/src/models/gemma4-assistant.cpp +++ b/src/models/gemma4-assistant.cpp @@ -131,15 +131,10 @@ llama_model_gemma4_assistant::graph::graph(const llm_graph_params & params) : ggml_tensor * inpL = cur; for (int il = 0; il < n_layer_nextn; ++il) { - const bool is_swa = hparams.is_swa(il); const int64_t n_embd_head = hparams.n_embd_head_k(il); const int64_t n_head = hparams.n_head(il); - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - const int n_rot_l = hparams.n_rot(il); - ggml_tensor * cur_norm = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); cb(cur_norm, "attn_norm", il); @@ -148,9 +143,7 @@ llama_model_gemma4_assistant::graph::graph(const llm_graph_params & params) : Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - ggml_tensor * freq_factors = is_swa ? nullptr : model.layers[il].rope_freqs; - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, freq_factors, n_rot_l, rope_type, n_ctx_orig, - freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); cb(Qcur, "Qcur_pos", il); cur = build_attn(inp_attn, model.layers[il].wo, nullptr, nullptr, diff --git a/src/models/gemma4.cpp b/src/models/gemma4.cpp index b937fc1fd..9c061310d 100644 --- a/src/models/gemma4.cpp +++ b/src/models/gemma4.cpp @@ -233,22 +233,12 @@ llama_model_gemma4::graph::graph(const llm_graph_params & params) : const int64_t n_head = hparams.n_head(il); const int64_t n_head_kv = hparams.n_head_kv(il); - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - const int n_rot_l = hparams.n_rot(il); - res->t_layer_inp[il] = inpL; // norm cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); - ggml_tensor * freq_factors = nullptr; - if (!hparams.is_swa(il)) { - // full_attention layers use rope_freqs for proportional rope - freq_factors = model.layers[il].rope_freqs; - } - // Q projection (shared for both non-KV and KV layers) // this is to mirror Gemma4Attention in pytorch code ggml_tensor * qkv_fused = nullptr; @@ -269,8 +259,7 @@ llama_model_gemma4::graph::graph(const llm_graph_params & params) : Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, freq_factors, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); cb(Qcur, "Qcur_pos", il); } @@ -303,8 +292,7 @@ llama_model_gemma4::graph::graph(const llm_graph_params & params) : cb(Kcur, "Kcur_normed", il); cb(Vcur, "Vcur_normed", il); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, freq_factors, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = build_rope(Kcur, inp_pos, il); cb(Kcur, "Kcur_pos", il); diff --git a/src/models/glm-dsa.cpp b/src/models/glm-dsa.cpp index f8d6956b3..358defe83 100644 --- a/src/models/glm-dsa.cpp +++ b/src/models/glm-dsa.cpp @@ -176,6 +176,14 @@ void llama_model_glm_dsa::load_arch_tensors(llama_model_loader & ml) { } } +llama_rope_params llama_model_glm_dsa::get_rope_params_indexer(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params_indexer(cparams, il); + + res.type = LLAMA_ROPE_TYPE_NORM; + + return res; +} + std::unique_ptr llama_model_glm_dsa::build_arch_graph(const llm_graph_params & params) const { if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { return std::make_unique(params); @@ -258,9 +266,7 @@ llama_model_glm_dsa::graph::graph(const llm_graph_params & params) : // {n_embd_indexer_head, n_indexer_head, n_tokens} indexer_q = ggml_reshape_3d(ctx0, indexer_q, n_embd_indexer_head, n_indexer_head, n_tokens); - indexer_q = ggml_rope_ext(ctx0, indexer_q, inp_pos, nullptr, n_rot, - LLAMA_ROPE_TYPE_NORM, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + indexer_q = build_rope_indexer(indexer_q, inp_pos, il); cb(indexer_q, "indexer_q", il); ggml_tensor * indexer_k = ggml_mul_mat(ctx0, model.layers[il].indexer_attn_k, cur); @@ -271,9 +277,7 @@ llama_model_glm_dsa::graph::graph(const llm_graph_params & params) : // {n_embd_indexer_head, 1, n_tokens} indexer_k = ggml_reshape_3d(ctx0, indexer_k, n_embd_indexer_head, 1, n_tokens); - indexer_k = ggml_rope_ext(ctx0, indexer_k, inp_pos, nullptr, n_rot, - LLAMA_ROPE_TYPE_NORM, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + indexer_k = build_rope_indexer(indexer_k, inp_pos, il); cb(indexer_k, "indexer_k", il); // perform Hadamard transform on indexer q and k @@ -387,12 +391,10 @@ llama_model_glm_dsa::graph::graph(const llm_graph_params & params) : ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); cb(k_pe, "k_pe", il); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + q_pe = build_rope(q_pe, inp_pos, il); cb(q_pe, "q_pe", il); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "k_pe", il); kv_cmpr = build_norm(kv_cmpr, model.layers[il].attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); @@ -647,12 +649,10 @@ llama_model_glm_dsa::graph_mtp::graph_mtp(const llm_graph_params & params) ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); cb(k_pe, "mtp_k_pe", il); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + q_pe = build_rope(q_pe, inp_pos, il); cb(q_pe, "mtp_q_pe", il); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "mtp_k_pe", il); kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); diff --git a/src/models/glm4-moe.cpp b/src/models/glm4-moe.cpp index 7bf186c5d..2d958aa95 100644 --- a/src/models/glm4-moe.cpp +++ b/src/models/glm4-moe.cpp @@ -203,13 +203,8 @@ llama_model_glm4_moe::graph_mtp::graph_mtp(const llm_graph_params & params) cb(Kcur, "mtp_Kcur_normed", il); } - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, - rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, - rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "mtp_Qcur", il); cb(Kcur, "mtp_Kcur", il); @@ -287,9 +282,6 @@ llama_model_glm4_moe::graph::graph(const llm_graph_params & params) : llm_graph_ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - ggml_tensor * cur; ggml_tensor * inpL; @@ -331,24 +323,9 @@ llama_model_glm4_moe::graph::graph(const llm_graph_params & params) : llm_graph_ cb(Kcur, "Kcur_normed", il); } - if (use_mrope) { - Qcur = ggml_rope_multi(ctx0, Qcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - } else { - // Normal RoPE - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, - rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, - rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - } + // Normal RoPE + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/glm4.cpp b/src/models/glm4.cpp index b7cd2b434..e57c51cbe 100644 --- a/src/models/glm4.cpp +++ b/src/models/glm4.cpp @@ -70,9 +70,6 @@ llama_model_glm4::graph::graph(const llm_graph_params & params) : llm_graph_cont GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - ggml_tensor * cur; ggml_tensor * inpL; @@ -105,24 +102,9 @@ llama_model_glm4::graph::graph(const llm_graph_params & params) : llm_graph_cont auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - if (use_mrope) { - Qcur = ggml_rope_multi(ctx0, Qcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - } else { - // Normal RoPE - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, - rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, - rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - } + // Normal RoPE + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/gptneox.cpp b/src/models/gptneox.cpp index ce211b18f..8217e5a8a 100644 --- a/src/models/gptneox.cpp +++ b/src/models/gptneox.cpp @@ -113,17 +113,8 @@ llama_model_gptneox::graph::graph(const llm_graph_params & params) : llm_graph_c auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/granite-hybrid.cpp b/src/models/granite-hybrid.cpp index 6d363f4cf..9e77e2efd 100644 --- a/src/models/granite-hybrid.cpp +++ b/src/models/granite-hybrid.cpp @@ -206,14 +206,8 @@ ggml_tensor * llama_model_granite_hybrid::graph::build_attention_layer(ggml_tens const int il) { auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il); - if (hparams.has_rope(il)) { - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/granite-swa.cpp b/src/models/granite-swa.cpp index 4ca4efed8..0276a8400 100644 --- a/src/models/granite-swa.cpp +++ b/src/models/granite-swa.cpp @@ -114,6 +114,16 @@ void llama_model_granite_swa::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_granite_swa::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + // the SWA layers use the global base and scale + res.freq_base = cparams.rope_freq_base; + res.freq_scale = cparams.rope_freq_scale; + + return res; +} + std::unique_ptr llama_model_granite_swa::build_arch_graph(const llm_graph_params & params) const { return std::make_unique(params); } @@ -204,21 +214,8 @@ ggml_tensor * llama_model_granite_swa::graph::build_attention_layer( auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il); - const bool use_rope = hparams.has_rope(il); - if (use_rope) { - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/granite-switch.cpp b/src/models/granite-switch.cpp index a972bc0e6..53195c97c 100644 --- a/src/models/granite-switch.cpp +++ b/src/models/granite-switch.cpp @@ -360,15 +360,8 @@ ggml_tensor * llama_model_granite_switch::graph::build_attention_layer( Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); - if (hparams.has_rope(il)) { - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); diff --git a/src/models/granite.cpp b/src/models/granite.cpp index 0c89c6949..d46aa7f2e 100644 --- a/src/models/granite.cpp +++ b/src/models/granite.cpp @@ -209,20 +209,8 @@ ggml_tensor * llama_model_granite::graph::build_attention_layer( auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il); - if (hparams.has_rope(il)) { - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/grok.cpp b/src/models/grok.cpp index e82c23f75..42486e60b 100644 --- a/src/models/grok.cpp +++ b/src/models/grok.cpp @@ -116,17 +116,8 @@ llama_model_grok::graph::graph(const llm_graph_params & params) : llm_graph_cont auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/grovemoe.cpp b/src/models/grovemoe.cpp index 44f44cecb..c86ddbbf5 100644 --- a/src/models/grovemoe.cpp +++ b/src/models/grovemoe.cpp @@ -100,14 +100,12 @@ llama_model_grovemoe::graph::graph(const llm_graph_params & params) : Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/hrm-text.cpp b/src/models/hrm-text.cpp index 5fe7e1542..8e3b34c57 100644 --- a/src/models/hrm-text.cpp +++ b/src/models/hrm-text.cpp @@ -114,12 +114,10 @@ ggml_tensor * llama_model_hrm_text::graph::build_stack(llm_graph_input_attn_kv * auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head_k, n_head, n_head_kv, s); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); cb(Qcur, "Qcur", s); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = build_rope(Kcur, inp_pos, il); cb(Kcur, "Kcur", s); cur = build_attn(inp_attn, diff --git a/src/models/hunyuan-dense.cpp b/src/models/hunyuan-dense.cpp index 3f9a75ba4..574a6412f 100644 --- a/src/models/hunyuan-dense.cpp +++ b/src/models/hunyuan-dense.cpp @@ -1,5 +1,13 @@ #include "models.h" +llama_rope_params llama_model_hunyuan_dense::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + res.k_norm = layers[il].attn_k_norm; + + return res; +} + std::unique_ptr llama_model_hunyuan_dense::build_arch_graph(const llm_graph_params & params) const { return std::make_unique(params); } diff --git a/src/models/hunyuan-moe.cpp b/src/models/hunyuan-moe.cpp index 2cfc17f8f..00659c88e 100644 --- a/src/models/hunyuan-moe.cpp +++ b/src/models/hunyuan-moe.cpp @@ -49,6 +49,14 @@ void llama_model_hunyuan_moe::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_hunyuan_moe::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + res.k_norm = layers[il].attn_k_norm; + + return res; +} + std::unique_ptr llama_model_hunyuan_moe::build_arch_graph(const llm_graph_params & params) const { return std::make_unique(params); } @@ -84,28 +92,17 @@ llama_model_hunyuan_moe::graph::graph(const llm_graph_params & params) : llm_gra // self-attention { - // rope freq factors for llama3; may return nullptr for llama2 and other models - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Kcur = build_rope(Kcur, inp_pos, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, diff --git a/src/models/hunyuan-vl.cpp b/src/models/hunyuan-vl.cpp index 52fc75ddf..219ba39bd 100644 --- a/src/models/hunyuan-vl.cpp +++ b/src/models/hunyuan-vl.cpp @@ -63,11 +63,6 @@ llama_model_hunyuan_vl::graph::graph(const llm_graph_params & params) : llm_grap GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); GGML_ASSERT(n_embd_head == n_rot); - const bool use_mrope = hparams.use_mrope(); - - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - ggml_tensor * cur; ggml_tensor * inpL; @@ -94,38 +89,12 @@ llama_model_hunyuan_vl::graph::graph(const llm_graph_params & params) : llm_grap cb(cur, "attn_norm", il); // self-attention { - // rope freq factors for llama3; may return nullptr for llama2 and other models - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - if (use_mrope) { - Qcur = ggml_rope_multi( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_multi( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - } else { - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/hy-v3.cpp b/src/models/hy-v3.cpp index 37150ef12..3fce6a246 100644 --- a/src/models/hy-v3.cpp +++ b/src/models/hy-v3.cpp @@ -119,19 +119,14 @@ llama_model_hy_v3::graph::graph(const llm_graph_params & params) : llm_graph_con // self-attention { - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cur = build_attn(inp_attn, model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s, @@ -286,19 +281,14 @@ llama_model_hy_v3::graph_mtp::graph_mtp(const llm_graph_params & params) cb(cur, "mtp_attn_norm", il); { - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head, n_head, n_head_kv, il); Qcur = build_norm(Qcur, layer.attn_q_norm, nullptr, LLM_NORM_RMS, il); Kcur = build_norm(Kcur, layer.attn_k_norm, nullptr, LLM_NORM_RMS, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); const float kq_scale = 1.0f / sqrtf(float(n_embd_head)); diff --git a/src/models/hy-v4.cpp b/src/models/hy-v4.cpp index 4e226bafd..386d504bc 100644 --- a/src/models/hy-v4.cpp +++ b/src/models/hy-v4.cpp @@ -154,6 +154,16 @@ void llama_model_hy_v4::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_hy_v4::get_rope_params_indexer(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params_indexer(cparams, il); + + // [nope | rope], the same as the MLA keys + res.n_offs = hparams.indexer_head_size - res.n_rot; + res.type = hparams.rope_type; + + return res; +} + std::unique_ptr llama_model_hy_v4::build_arch_graph(const llm_graph_params & params) const { return std::make_unique(params); } @@ -297,11 +307,9 @@ ggml_tensor * llama_model_hy_v4::graph::build_attention( ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + q_pe = build_rope(q_pe, inp_pos, il); cb(q_pe, "q_pe", il); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "k_pe", il); kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); @@ -347,17 +355,13 @@ ggml_tensor * llama_model_hy_v4::graph::build_indexer_top_k( const int64_t n_indexer_head = hparams.indexer_n_head; const int64_t n_embd_indexer = hparams.indexer_head_size; - const int64_t n_embd_indexer_rope = hparams.n_rot(); - const int64_t n_embd_indexer_nope = n_embd_indexer - n_embd_indexer_rope; // nope rows come first, so rope only the last n_embd_indexer_rope rows, same as the MLA path ggml_tensor * iq = ggml_mul_mat(ctx0, layer.indexer_attn_q_b, qr); iq = ggml_reshape_3d(ctx0, iq, n_embd_indexer, n_indexer_head, n_tokens); - iq = ggml_rope_ext(ctx0, iq, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, - freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - iq = ggml_rope_set_offset(iq, n_embd_indexer_nope); + iq = build_rope_indexer(iq, inp_pos, il); cb(iq, "indexer_q", il); ggml_tensor * ik = ggml_mul_mat(ctx0, layer.indexer_attn_k, cur); @@ -366,9 +370,7 @@ ggml_tensor * llama_model_hy_v4::graph::build_indexer_top_k( ik = ggml_reshape_3d(ctx0, ik, n_embd_indexer, 1, n_tokens); - ik = ggml_rope_ext(ctx0, ik, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, - freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - ik = ggml_rope_set_offset(ik, n_embd_indexer_nope); + ik = build_rope_indexer(ik, inp_pos, il); cb(ik, "indexer_k", il); // the reference applies a Hadamard rotation here, but it only helps its FP8 kernels. @@ -456,11 +458,9 @@ llama_model_hy_v4::graph::build_attention_dsa( ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + q_pe = build_rope(q_pe, inp_pos, il); cb(q_pe, "q_pe", il); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "k_pe", il); kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); diff --git a/src/models/internlm2.cpp b/src/models/internlm2.cpp index 4f7fb4346..39ce67af6 100644 --- a/src/models/internlm2.cpp +++ b/src/models/internlm2.cpp @@ -71,17 +71,8 @@ llama_model_internlm2::graph::graph(const llm_graph_params & params) : llm_graph auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/jais2.cpp b/src/models/jais2.cpp index 8cbdcaff4..8376423ea 100644 --- a/src/models/jais2.cpp +++ b/src/models/jais2.cpp @@ -84,17 +84,8 @@ llama_model_jais2::graph::graph(const llm_graph_params & params) : llm_graph_con n_embd_head, n_head, n_head_kv, il); // Apply RoPE - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur_rope", il); cb(Kcur, "Kcur_rope", il); diff --git a/src/models/k2-horizon.cpp b/src/models/k2-horizon.cpp index 487e7fc3a..534efa523 100644 --- a/src/models/k2-horizon.cpp +++ b/src/models/k2-horizon.cpp @@ -261,12 +261,8 @@ llama_model_k2_horizon::graph::graph(const llm_graph_params & params) : llm_grap Kcur = build_norm(Kcur, layer.attn_k_norm, NULL, LLM_NORM_RMS, il); } - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/kolibri1.cpp b/src/models/kolibri1.cpp index bf11fba3c..c4d8eca1b 100644 --- a/src/models/kolibri1.cpp +++ b/src/models/kolibri1.cpp @@ -15,6 +15,7 @@ void llama_model_kolibri1::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); + hparams.rope_pattern = hparams.is_swa_impl; hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; @@ -95,22 +96,11 @@ llama_model_kolibri1::graph::graph(const llm_graph_params & params) cb(Qcur, "Qcur_normed", il); cb(Kcur, "Kcur_normed", il); - if (hparams.is_swa(il)) { - const float freq_base_l = model.get_rope_freq_base (cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); + Qcur = build_rope(Qcur, inp_pos, il); + cb(Qcur, "Qcur_rope", il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Qcur, "Qcur_rope", il); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Kcur, "Kcur_rope", il); - } + Kcur = build_rope(Kcur, inp_pos, il); + cb(Kcur, "Kcur_rope", il); cur = build_attn(inp_attn, layer.wo, NULL, layer.wo_s, diff --git a/src/models/laguna.cpp b/src/models/laguna.cpp index bc399670b..1bf25eacf 100644 --- a/src/models/laguna.cpp +++ b/src/models/laguna.cpp @@ -148,6 +148,21 @@ void llama_model_laguna::load_arch_tensors(llama_model_loader & ml) { } } +llama_rope_params llama_model_laguna::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + // plain RoPE on the SWA layers + if (hparams.is_swa(il)) { + res.n_ctx_orig = hparams.n_ctx_train; + res.ext_factor = 0.0f; + res.attn_factor = 1.0f; + res.beta_fast = 0.0f; + res.beta_slow = 0.0f; + } + + return res; +} + std::unique_ptr llama_model_laguna::build_arch_graph(const llm_graph_params & params) const { return std::make_unique(params); } @@ -173,26 +188,11 @@ llama_model_laguna::graph::graph(const llm_graph_params & params) : llm_graph_co const float kq_scale = 1.0f / sqrtf(float(n_embd_head)); for (int il = 0; il < n_layer; ++il) { - const bool is_swa_il = hparams.is_swa(il); const int64_t n_head_il = hparams.n_head(il); const int64_t n_head_kv_il = hparams.n_head_kv(il); // Per-layer-type RoPE config. SWA layers run plain rope (no YaRN), // achieved by zeroing the YaRN ext/beta params for those layers. - const int n_rot_l = is_swa_il ? hparams.n_rot_swa : n_rot; - const float freq_base_l = is_swa_il ? hparams.rope_freq_base_train_swa : freq_base; - const float freq_scale_l = is_swa_il ? hparams.rope_freq_scale_train_swa : freq_scale; - const float ext_factor_l = is_swa_il ? 0.0f : ext_factor; - // YaRN magnitude scaling (mscale) is already handled by the framework: - // llama_context pre-divides cparams.yarn_attn_factor by (1 + 0.1*ln(factor)) - // to cancel ggml rope_yarn's internal mscale *= 1 + 0.1*ln(1/freq_scale). - // Pass attn_factor straight through (like every other arch); SWA layers run - // plain RoPE (ext_factor 0, no mscale) so force 1.0 there. - const float attn_factor_l = is_swa_il ? 1.0f : attn_factor; - const float beta_fast_l = is_swa_il ? 0.0f : beta_fast; - const float beta_slow_l = is_swa_il ? 0.0f : beta_slow; - const int n_ctx_orig_l = is_swa_il ? hparams.n_ctx_train : n_ctx_orig; - ggml_tensor * inpSA = inpL; // Pre-norm @@ -218,12 +218,8 @@ llama_model_laguna::graph::graph(const llm_graph_params & params) : llm_graph_co cb(Qcur, "Qcur_normed", il); cb(Kcur, "Kcur_normed", il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, - n_rot_l, rope_type, n_ctx_orig_l, freq_base_l, freq_scale_l, - ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, - n_rot_l, rope_type, n_ctx_orig_l, freq_base_l, freq_scale_l, - ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur_rope", il); cb(Kcur, "Kcur_rope", il); diff --git a/src/models/lfm2.cpp b/src/models/lfm2.cpp index 9d83b3dc1..86b948cf3 100644 --- a/src/models/lfm2.cpp +++ b/src/models/lfm2.cpp @@ -134,6 +134,16 @@ void llama_model_lfm2::load_arch_tensors(llama_model_loader &) { dense_2_out_layers_b = create_tensor(tn(LLM_TENSOR_DENSE_2_OUT, "bias"), {hparams.n_embd_out() }, TENSOR_NOT_REQUIRED); } +llama_rope_params llama_model_lfm2::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + // the SWA layers use the global base and scale + res.freq_base = cparams.rope_freq_base; + res.freq_scale = cparams.rope_freq_scale; + + return res; +} + std::unique_ptr llama_model_lfm2::build_arch_graph(const llm_graph_params & params) const { if (hparams.n_layer_decision > 0) { return std::make_unique(params); @@ -170,10 +180,8 @@ llama_model_lfm2::graph::graph(const llm_graph_params & params) : cb(k, "model.layers.{}.self_attn.k_layernorm", il); // RoPE - q = ggml_rope_ext(ctx0, q, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, - attn_factor, beta_fast, beta_slow); - k = ggml_rope_ext(ctx0, k, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, - attn_factor, beta_fast, beta_slow); + q = build_rope(q, inp_pos, il); + k = build_rope(k, inp_pos, il); cur = build_attn(inp_attn, model.layers[il].wo, NULL, model.layers[il].wo_s, @@ -384,10 +392,8 @@ llama_model_lfm2::graph_decision::graph_decision(const llm_graph_params & params q = build_norm(q, layer.attn_q_norm, NULL, LLM_NORM_RMS, il); k = build_norm(k, layer.attn_k_norm, NULL, LLM_NORM_RMS, il); - q = ggml_rope_ext(ctx0, q, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, - attn_factor, beta_fast, beta_slow); - k = ggml_rope_ext(ctx0, k, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, - attn_factor, beta_fast, beta_slow); + q = build_rope(q, inp_pos, il); + k = build_rope(k, inp_pos, il); cur = build_attn(inp_attn[0], layer.wo, NULL, layer.wo_s, diff --git a/src/models/lfm3.cpp b/src/models/lfm3.cpp index 9c5399c2f..0746394ce 100644 --- a/src/models/lfm3.cpp +++ b/src/models/lfm3.cpp @@ -106,12 +106,6 @@ llama_model_lfm3::graph::graph(const llm_graph_params & params) : const int64_t n_head_il = hparams.n_head(il); const int64_t n_head_kv = hparams.n_head_kv(il); - const int n_rot_l = hparams.n_rot(il); - const float freq_base_l = model.get_rope_freq_base (cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - - ggml_tensor * rope_factors = hparams.is_swa(il) ? nullptr : model.layers[il].rope_freqs; - auto [q, k, v] = build_qkv(model.layers[il], cur_in, n_embd_head, n_head_il, n_head_kv, il); q = build_norm(q, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); @@ -119,10 +113,8 @@ llama_model_lfm3::graph::graph(const llm_graph_params & params) : k = build_norm(k, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(k, "model.layers.{}.self_attn.k_layernorm", il); - q = ggml_rope_ext(ctx0, q, inp_pos, rope_factors, n_rot_l, rope_type, n_ctx_orig, - freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow); - k = ggml_rope_ext(ctx0, k, inp_pos, rope_factors, n_rot_l, rope_type, n_ctx_orig, - freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow); + q = build_rope(q, inp_pos, il); + k = build_rope(k, inp_pos, il); ggml_tensor * attn_out = build_attn(inp_attn, NULL, NULL, NULL, diff --git a/src/models/limite.cpp b/src/models/limite.cpp index 67b368cfd..2bd45561a 100644 --- a/src/models/limite.cpp +++ b/src/models/limite.cpp @@ -13,6 +13,7 @@ void llama_model_limite::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); load_swa_pattern(ml, 4); + hparams.rope_pattern = hparams.is_swa_impl; std::fill(hparams.mudd_tap_idx.begin(), hparams.mudd_tap_idx.end(), -1); ml.get_key(LLM_KV_MUDD_TAP_COUNT, hparams.n_mudd_taps, false); @@ -68,6 +69,16 @@ void llama_model_limite::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_limite::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + // the SWA layers use the global base and scale + res.freq_base = cparams.rope_freq_base; + res.freq_scale = cparams.rope_freq_scale; + + return res; +} + std::unique_ptr llama_model_limite::build_arch_graph(const llm_graph_params & params) const { return std::make_unique(params); } @@ -160,14 +171,10 @@ llama_model_limite::graph::graph(const llm_graph_params & params) : llm_graph_co cb(Kcur, "Kcur_normed", il); // the full attention layers use no positional encoding - if (hparams.is_swa(il)) { - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); cur = build_attn(inp_attn, nullptr, nullptr, nullptr, diff --git a/src/models/llada-moe.cpp b/src/models/llada-moe.cpp index d7baf7a4b..0086d52af 100644 --- a/src/models/llada-moe.cpp +++ b/src/models/llada-moe.cpp @@ -90,17 +90,8 @@ llama_model_llada_moe::graph::graph(const llm_graph_params & params) : llm_graph Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/llada.cpp b/src/models/llada.cpp index 019966a37..8777cff64 100644 --- a/src/models/llada.cpp +++ b/src/models/llada.cpp @@ -59,6 +59,14 @@ void llama_model_llada::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_llada::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + res.factors = nullptr; + + return res; +} + std::unique_ptr llama_model_llada::build_arch_graph(const llm_graph_params & params) const { return std::make_unique(params); } @@ -96,11 +104,8 @@ llama_model_llada::graph::graph(const llm_graph_params & params) : llm_graph_con auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/llama.cpp b/src/models/llama.cpp index 87174c6bc..26d54968c 100644 --- a/src/models/llama.cpp +++ b/src/models/llama.cpp @@ -136,24 +136,12 @@ llama_model_llama::graph::graph(const llm_graph_params & params) : llm_gr // self-attention { - // rope freq factors for llama3; may return nullptr for llama2 and other models - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/llama4.cpp b/src/models/llama4.cpp index 433d1ef58..5fa2a623f 100644 --- a/src/models/llama4.cpp +++ b/src/models/llama4.cpp @@ -23,6 +23,10 @@ void llama_model_llama4::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); } + for (uint32_t il = 0; il < hparams.n_layer_all; ++il) { + hparams.rope_pattern[il] = (il + 1) % hparams.n_no_rope_layer_step != 0; + } + switch (hparams.n_expert) { case 0: { // MobileLLM (no MoE) @@ -134,8 +138,6 @@ llama_model_llama4::graph::graph(const llm_graph_params & params) : llm_gr ggml_tensor * inp_out_ids = build_inp_out_ids(); for (int il = 0; il < n_layer; ++il) { - const float freq_base_l = model.get_rope_freq_base (cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); ggml_tensor * inpSA = inpL; @@ -151,25 +153,13 @@ llama_model_llama4::graph::graph(const llm_graph_params & params) : llm_gr // self-attention { - // rope freq factors for llama3; may return nullptr for llama2 and other models - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); if (use_rope) { - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); } else if (inp_attn_scale) { Qcur = ggml_mul(ctx0, Qcur, inp_attn_scale); } diff --git a/src/models/maincoder.cpp b/src/models/maincoder.cpp index c9bf837c1..1a2d2c695 100644 --- a/src/models/maincoder.cpp +++ b/src/models/maincoder.cpp @@ -40,6 +40,14 @@ void llama_model_maincoder::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_maincoder::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + res.k_norm = layers[il].attn_k_norm; + + return res; +} + std::unique_ptr llama_model_maincoder::build_arch_graph(const llm_graph_params & params) const { return std::make_unique(params); } @@ -77,17 +85,8 @@ llama_model_maincoder::graph::graph(const llm_graph_params & params) : llm_graph auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); diff --git a/src/models/maion-coder.cpp b/src/models/maion-coder.cpp index 1c778e06a..5c1e376c5 100644 --- a/src/models/maion-coder.cpp +++ b/src/models/maion-coder.cpp @@ -94,12 +94,8 @@ llama_model_maion_coder::graph::graph(const llm_graph_params & params) : llm_gra Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); // Full-attention layers do not use RoPE. - if (hparams.is_swa(il)) { - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cur = build_attn(inp_attn, model.layers[il].wo, NULL, model.layers[il].wo_s, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); diff --git a/src/models/maple.cpp b/src/models/maple.cpp index d853d601c..eb0b3b4ac 100644 --- a/src/models/maple.cpp +++ b/src/models/maple.cpp @@ -8,6 +8,7 @@ void llama_model_maple::load_arch_hparams(llama_model_loader & ml) { ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); + hparams.rope_pattern = hparams.is_swa_impl; hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; @@ -87,16 +88,8 @@ llama_model_maple::graph::graph(const llm_graph_params & params) : cb(Qcur, "Qcur_normed", il); cb(Kcur, "Kcur_normed", il); - if (hparams.is_swa(il)) { - const int64_t n_rot_l = hparams.n_rot(il); - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot_l, rope_type, n_ctx_orig, freq_base_l, - freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot_l, rope_type, n_ctx_orig, freq_base_l, - freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); diff --git a/src/models/mellum.cpp b/src/models/mellum.cpp index e25711b2c..83f70019d 100644 --- a/src/models/mellum.cpp +++ b/src/models/mellum.cpp @@ -88,6 +88,19 @@ void llama_model_mellum::load_arch_tensors(llama_model_loader & ml) { } } +llama_rope_params llama_model_mellum::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + if (hparams.is_swa(il)) { + res.ext_factor = 0.0f; + res.attn_factor = 1.0f; + res.beta_fast = 32.0f; + res.beta_slow = 1.0f; + } + + return res; +} + std::unique_ptr llama_model_mellum::build_arch_graph(const llm_graph_params & params) const { const bool iswa = hparams.swa_type == LLAMA_SWA_TYPE_STANDARD; @@ -151,26 +164,8 @@ llama_model_mellum::graph::graph(const llm_graph_params & params) : llm_gr Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - const bool is_swa = hparams.is_swa(il); - - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - const float ext_factor_l = is_swa ? 0.0f : ext_factor; - const float attn_factor_l = is_swa ? 1.0f : attn_factor; - const float beta_fast_l = is_swa ? 32.0f : beta_fast; - const float beta_slow_l = is_swa ? 1.0f : beta_slow; - - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); @@ -329,26 +324,8 @@ llama_model_mellum::graph_mtp::graph_mtp(const llm_graph_params & params) Kcur = build_norm(Kcur, layer.attn_k_norm, nullptr, LLM_NORM_RMS, il); cb(Kcur, "mtp_Kcur_normed", il); - const bool is_swa = hparams.is_swa(il); - - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - const float ext_factor_l = is_swa ? 0.0f : ext_factor; - const float attn_factor_l = is_swa ? 1.0f : attn_factor; - const float beta_fast_l = is_swa ? 32.0f : beta_fast; - const float beta_slow_l = is_swa ? 1.0f : beta_slow; - - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "mtp_Qcur", il); cb(Kcur, "mtp_Kcur", il); diff --git a/src/models/mimo2.cpp b/src/models/mimo2.cpp index a86b0f96d..b318b05ed 100644 --- a/src/models/mimo2.cpp +++ b/src/models/mimo2.cpp @@ -104,8 +104,6 @@ llama_model_mimo2::graph::graph(const llm_graph_params & params) : llm_graph_con uint32_t n_head_l = hparams.n_head(il); uint32_t n_head_kv_l = hparams.n_head_kv(il); - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); cur = inpL; @@ -148,17 +146,8 @@ llama_model_mimo2::graph::graph(const llm_graph_params & params) : llm_graph_con Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head_v, n_head_kv_l, n_tokens); } - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); @@ -276,8 +265,6 @@ llama_model_mimo2::graph_mtp::graph_mtp(const llm_graph_params & params) const uint32_t n_head_l = hparams.n_head(il); const uint32_t n_head_kv_l = hparams.n_head_kv(il); - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); const float v_scale = hparams.f_attn_value_scale; auto inp = std::make_unique(hparams.n_embd_inp(), hparams.n_embd); @@ -333,15 +320,8 @@ llama_model_mimo2::graph_mtp::graph_mtp(const llm_graph_params & params) ggml_tensor * Kcur = ggml_view_3d(ctx0, qkv, n_embd_head_k, n_head_kv_l, n_tokens, row_k, row_full, k_off); ggml_tensor * Vcur = ggml_view_3d(ctx0, qkv, n_embd_head_v, n_head_kv_l, n_tokens, row_v, row_full, v_off); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "mtp_Qcur", il); cb(Kcur, "mtp_Kcur", il); diff --git a/src/models/minicpm3.cpp b/src/models/minicpm3.cpp index e816c66ee..67c8d6708 100644 --- a/src/models/minicpm3.cpp +++ b/src/models/minicpm3.cpp @@ -91,8 +91,6 @@ llama_model_minicpm3::graph::graph(const llm_graph_params & params) : llm_graph_ for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // norm cur = build_norm(inpL, model.layers[il].attn_norm, NULL, @@ -162,20 +160,12 @@ llama_model_minicpm3::graph::graph(const llm_graph_params & params) : llm_graph_ v_states = ggml_cont(ctx0, v_states); cb(v_states, "v_states", il); - q = ggml_rope_ext( - ctx0, q, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + q = build_rope(q, inp_pos, il); q = ggml_rope_set_offset(q, n_embd_head_qk_nope); cb(q, "q_rope", il); // shared RoPE key - k_pe = ggml_rope_ext( - ctx0, k_pe, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "k_pe", il); ggml_tensor * q_states = q; diff --git a/src/models/minimax-01.cpp b/src/models/minimax-01.cpp index 7a2e55431..08268e0f3 100644 --- a/src/models/minimax-01.cpp +++ b/src/models/minimax-01.cpp @@ -255,17 +255,8 @@ llama_model_minimax_01::graph::graph(const llm_graph_params & params) : llm_grap auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/minimax-m2.cpp b/src/models/minimax-m2.cpp index 46fffb101..5b8258255 100644 --- a/src/models/minimax-m2.cpp +++ b/src/models/minimax-m2.cpp @@ -92,17 +92,8 @@ llama_model_minimax_m2::graph::graph(const llm_graph_params & params) : llm_grap Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/minimax-m3.cpp b/src/models/minimax-m3.cpp index 4b326896f..2002d08be 100644 --- a/src/models/minimax-m3.cpp +++ b/src/models/minimax-m3.cpp @@ -318,14 +318,8 @@ llama_model_minimax_m3::graph::graph(const llm_graph_params & params) : llm_grap cb(Kcur, "Kcur_normed", il); // partial rotary: only the first n_rot dims are rotated - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); @@ -347,10 +341,8 @@ llama_model_minimax_m3::graph::graph(const llm_graph_params & params) : llm_grap ik = ggml_reshape_3d(ctx0, ik, n_idx_dim, 1, n_tokens); iq = build_norm(iq, model.layers[il].index_q_norm, NULL, LLM_NORM_RMS, il); // +1 baked ik = build_norm(ik, model.layers[il].index_k_norm, NULL, LLM_NORM_RMS, il); - iq = ggml_rope_ext(ctx0, iq, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - ik = ggml_rope_ext(ctx0, ik, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + iq = build_rope(iq, inp_pos, il); + ik = build_rope(ik, inp_pos, il); const auto * mctx_msa_l = static_cast(mctx); const auto * mctx_cur = mctx_msa_l->get_base(); diff --git a/src/models/mistral-large4.cpp b/src/models/mistral-large4.cpp index b7a6c4279..a4315a991 100644 --- a/src/models/mistral-large4.cpp +++ b/src/models/mistral-large4.cpp @@ -148,6 +148,15 @@ void llama_model_mistral_large4::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_mistral_large4::get_rope_params_indexer(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params_indexer(cparams, il); + + res.type = hparams.rope_type; + res.factors = get_rope_factors(cparams, il); + + return res; +} + std::unique_ptr llama_model_mistral_large4::build_arch_graph(const llm_graph_params & params) const { return std::make_unique(params); } @@ -216,8 +225,6 @@ llama_model_mistral_large4::graph::graph(const llm_graph_params & params) : const int64_t n_rot_l = use_rope ? n_rot : 0; const int64_t n_embd_head_qk_nope = n_embd_head_k - n_rot_l; - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - ggml_tensor * inpSA = inpL; cur = build_norm(inpL, layer.attn_norm, NULL, LLM_NORM_RMS, il); @@ -250,14 +257,10 @@ llama_model_mistral_large4::graph::graph(const llm_graph_params & params) : indexer_k = ggml_reshape_3d(ctx0, indexer_k, n_embd_indexer_head, 1, n_tokens); if (use_rope) { - indexer_q = ggml_rope_ext(ctx0, indexer_q, inp_pos, rope_factors, n_rot, - rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + indexer_q = build_rope_indexer(indexer_q, inp_pos, il); cb(indexer_q, "indexer_q", il); - indexer_k = ggml_rope_ext(ctx0, indexer_k, inp_pos, rope_factors, n_rot, - rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + indexer_k = build_rope_indexer(indexer_k, inp_pos, il); cb(indexer_k, "indexer_k", il); } @@ -368,8 +371,7 @@ llama_model_mistral_large4::graph::graph(const llm_graph_params & params) : ggml_row_size(q->type, n_embd_head_k), ggml_row_size(q->type, n_embd_head_k) * n_head, ggml_row_size(q->type, n_embd_head_qk_nope)); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + q_pe = build_rope(q_pe, inp_pos, il); cb(q_pe, "q_pe", il); Qcur = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0); @@ -406,8 +408,7 @@ llama_model_mistral_large4::graph::graph(const llm_graph_params & params) : ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_rot), ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_rot), ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "k_pe", il); Kcur = ggml_concat(ctx0, kv_cmpr, k_pe, 0); diff --git a/src/models/mistral3.cpp b/src/models/mistral3.cpp index 3686d9c21..ab8121863 100644 --- a/src/models/mistral3.cpp +++ b/src/models/mistral3.cpp @@ -175,24 +175,12 @@ llama_model_mistral3::graph::graph(const llm_graph_params & params) : llm_graph_ // self-attention { - // rope freq factors for llama3; may return nullptr for llama2 and other models - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); - // compute Q and K and RoPE them auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/models.h b/src/models/models.h index 4c8f85918..3e338c26b 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -174,7 +174,6 @@ struct llama_model_llama : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_llama4 : public llama_model_base { llama_model_llama4(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -188,7 +187,6 @@ struct llama_model_llama4 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_llama_embed : public llama_model_llama { llama_model_llama_embed(const struct llama_model_params & params) : llama_model_llama(params) {} // reuse load_arch_hparams and load_arch_tensors from llama_model_llama @@ -199,7 +197,6 @@ struct llama_model_llama_embed : public llama_model_llama { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_maincoder : public llama_model_base { llama_model_maincoder(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -209,10 +206,10 @@ struct llama_model_maincoder : public llama_model_base { graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_maion_coder : public llama_model_base { llama_model_maion_coder(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -225,7 +222,6 @@ struct llama_model_maion_coder : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_talkie : public llama_model_base { llama_model_talkie(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -238,7 +234,6 @@ struct llama_model_talkie : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_deci : public llama_model_base { llama_model_deci(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -251,7 +246,6 @@ struct llama_model_deci : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_baichuan : public llama_model_base { llama_model_baichuan(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -264,7 +258,6 @@ struct llama_model_baichuan : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_falcon : public llama_model_base { llama_model_falcon(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -277,7 +270,6 @@ struct llama_model_falcon : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_grok : public llama_model_base { llama_model_grok(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -290,7 +282,6 @@ struct llama_model_grok : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_starcoder : public llama_model_base { llama_model_starcoder(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -303,7 +294,6 @@ struct llama_model_starcoder : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_refact : public llama_model_base { llama_model_refact(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -316,7 +306,6 @@ struct llama_model_refact : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_bert : public llama_model_base { llama_model_bert(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -329,7 +318,6 @@ struct llama_model_bert : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_jina_bert_v2 : public llama_model_base { llama_model_jina_bert_v2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -340,7 +328,6 @@ struct llama_model_jina_bert_v2 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_jina_bert_v3 : public llama_model_base { llama_model_jina_bert_v3(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -351,7 +338,6 @@ struct llama_model_jina_bert_v3 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_nomic_bert : public llama_model_base { llama_model_nomic_bert(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -362,7 +348,6 @@ struct llama_model_nomic_bert : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_nomic_bert_moe : public llama_model_base { llama_model_nomic_bert_moe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -373,7 +358,6 @@ struct llama_model_nomic_bert_moe : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_modern_bert : public llama_model_base { llama_model_modern_bert(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -391,7 +375,6 @@ struct llama_model_modern_bert : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_neo_bert : public llama_model_base { llama_model_neo_bert(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -404,7 +387,6 @@ struct llama_model_neo_bert : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_eurobert : public llama_model_base { llama_model_eurobert(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -417,7 +399,6 @@ struct llama_model_eurobert : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_bloom : public llama_model_base { llama_model_bloom(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -430,7 +411,6 @@ struct llama_model_bloom : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - // Quant-only stub for mmproj GGUFs // none of these are ever called, they only exist to satisfy the llama_model_base interface struct llama_model_clip : public llama_model_base { @@ -446,7 +426,6 @@ struct llama_model_clip : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_mpt : public llama_model_base { llama_model_mpt(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -459,7 +438,6 @@ struct llama_model_mpt : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_stablelm : public llama_model_base { llama_model_stablelm(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -487,6 +465,7 @@ struct llama_model_mellum : public llama_model_base { graph_mtp(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; @@ -518,7 +497,6 @@ struct llama_model_qwen : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_qwen2 : public llama_model_base { llama_model_qwen2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -531,7 +509,6 @@ struct llama_model_qwen2 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_dream : public llama_model_base { llama_model_dream(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -544,7 +521,6 @@ struct llama_model_dream : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_llada : public llama_model_base { llama_model_llada(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -554,10 +530,10 @@ struct llama_model_llada : public llama_model_base { graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_llada_moe : public llama_model_base { llama_model_llada_moe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -570,7 +546,6 @@ struct llama_model_llada_moe : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_rnd1 : public llama_model_base { llama_model_rnd1(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -583,7 +558,6 @@ struct llama_model_rnd1 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_qwen2vl : public llama_model_base { llama_model_qwen2vl(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -596,7 +570,6 @@ struct llama_model_qwen2vl : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_qwen2moe : public llama_model_base { llama_model_qwen2moe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -609,7 +582,6 @@ struct llama_model_qwen2moe : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_sensenova_u1 : public llama_model_base { llama_model_sensenova_u1(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -624,7 +596,6 @@ struct llama_model_sensenova_u1 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_qwen3 : public llama_model_base { llama_model_qwen3(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -675,7 +646,6 @@ struct llama_model_vllm_decision2 : public llama_model_qwen3 { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_qwen3moe : public llama_model_base { llama_model_qwen3moe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -688,7 +658,6 @@ struct llama_model_qwen3moe : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_flint : public llama_model_base { llama_model_flint(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -698,10 +667,10 @@ struct llama_model_flint : public llama_model_base { graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_qwen3vl : public llama_model_base { llama_model_qwen3vl(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -714,7 +683,6 @@ struct llama_model_qwen3vl : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_qwen3vlmoe : public llama_model_base { llama_model_qwen3vlmoe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -727,12 +695,10 @@ struct llama_model_qwen3vlmoe : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_qwen3tts : public llama_model_qwen3vl { llama_model_qwen3tts(const struct llama_model_params & params) : llama_model_qwen3vl(params) {} }; - struct llama_model_phi2 : public llama_model_base { llama_model_phi2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -745,7 +711,6 @@ struct llama_model_phi2 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_phi3 : public llama_model_base { llama_model_phi3(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -759,7 +724,6 @@ struct llama_model_phi3 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_phimoe : public llama_model_base { llama_model_phimoe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -771,7 +735,6 @@ struct llama_model_phimoe : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_plamo : public llama_model_base { llama_model_plamo(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -784,7 +747,6 @@ struct llama_model_plamo : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_plamo2 : public llama_model_base { llama_model_plamo2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -801,7 +763,6 @@ struct llama_model_plamo2 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_plamo3 : public llama_model_base { llama_model_plamo3(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -815,7 +776,6 @@ struct llama_model_plamo3 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_gpt2 : public llama_model_base { llama_model_gpt2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -828,7 +788,6 @@ struct llama_model_gpt2 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_pockettts : public llama_model_base { llama_model_pockettts(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -841,7 +800,6 @@ struct llama_model_pockettts : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_codeshell : public llama_model_base { llama_model_codeshell(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -854,7 +812,6 @@ struct llama_model_codeshell : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_orion : public llama_model_base { llama_model_orion(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -867,7 +824,6 @@ struct llama_model_orion : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_internlm2 : public llama_model_base { llama_model_internlm2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -880,7 +836,6 @@ struct llama_model_internlm2 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_minicpm3 : public llama_model_base { llama_model_minicpm3(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -893,7 +848,6 @@ struct llama_model_minicpm3 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_gemma : public llama_model_base { llama_model_gemma(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -906,7 +860,6 @@ struct llama_model_gemma : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_gemma2 : public llama_model_base { llama_model_gemma2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -919,7 +872,6 @@ struct llama_model_gemma2 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_gemma3 : public llama_model_base { llama_model_gemma3(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -933,7 +885,6 @@ struct llama_model_gemma3 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_gemma3n : public llama_model_base { llama_model_gemma3n(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -964,7 +915,6 @@ struct llama_model_gemma3n : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_gemma4 : public llama_model_base { llama_model_gemma4(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -983,7 +933,6 @@ struct llama_model_gemma4 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_gemma4_assistant : public llama_model_base { llama_model_gemma4_assistant(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -996,7 +945,6 @@ struct llama_model_gemma4_assistant : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_gemma_embedding : public llama_model_base { llama_model_gemma_embedding(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1009,7 +957,6 @@ struct llama_model_gemma_embedding : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_gemma_embedding2 : public llama_model_base { llama_model_gemma_embedding2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1024,7 +971,6 @@ struct llama_model_gemma_embedding2 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_starcoder2 : public llama_model_base { llama_model_starcoder2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1037,7 +983,6 @@ struct llama_model_starcoder2 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_mamba : public llama_model_base { llama_model_mamba(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1050,7 +995,6 @@ struct llama_model_mamba : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_mamba2 : public llama_model_base { llama_model_mamba2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1061,7 +1005,6 @@ struct llama_model_mamba2 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_maple : public llama_model_base { llama_model_maple(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1074,7 +1017,6 @@ struct llama_model_maple : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_jamba : public llama_model_base { llama_model_jamba(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1087,7 +1029,6 @@ struct llama_model_jamba : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_xverse : public llama_model_base { llama_model_xverse(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1100,7 +1041,6 @@ struct llama_model_xverse : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_command_r : public llama_model_base { llama_model_command_r(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1113,7 +1053,6 @@ struct llama_model_command_r : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_cohere2 : public llama_model_base { llama_model_cohere2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1123,10 +1062,10 @@ struct llama_model_cohere2 : public llama_model_base { graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_cohere2moe : public llama_model_base { llama_model_cohere2moe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1140,10 +1079,10 @@ struct llama_model_cohere2moe : public llama_model_base { graph_mtp(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_dbrx : public llama_model_base { llama_model_dbrx(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1156,7 +1095,6 @@ struct llama_model_dbrx : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_olmo : public llama_model_base { llama_model_olmo(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1169,7 +1107,6 @@ struct llama_model_olmo : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_olmo2 : public llama_model_base { llama_model_olmo2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1180,10 +1117,10 @@ struct llama_model_olmo2 : public llama_model_base { graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_olmoe : public llama_model_base { llama_model_olmoe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1196,7 +1133,6 @@ struct llama_model_olmoe : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_muse_spark : public llama_model_base { llama_model_muse_spark(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1209,7 +1145,6 @@ struct llama_model_muse_spark : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_muse_glimmer : public llama_model_base { llama_model_muse_glimmer(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1222,7 +1157,6 @@ struct llama_model_muse_glimmer : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_openelm : public llama_model_base { llama_model_openelm(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1235,7 +1169,6 @@ struct llama_model_openelm : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_gptneox : public llama_model_base { llama_model_gptneox(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1248,7 +1181,6 @@ struct llama_model_gptneox : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_arctic : public llama_model_base { llama_model_arctic(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1261,7 +1193,6 @@ struct llama_model_arctic : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_deepseek : public llama_model_base { llama_model_deepseek(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1274,7 +1205,6 @@ struct llama_model_deepseek : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_deepseek2 : public llama_model_base { llama_model_deepseek2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1291,7 +1221,6 @@ struct llama_model_deepseek2 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_deepseek32 : public llama_model_base { llama_model_deepseek32(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1308,7 +1237,6 @@ struct llama_model_deepseek32 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_dots3note : public llama_model_base { llama_model_dots3note(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1318,6 +1246,7 @@ struct llama_model_dots3note : public llama_model_base { graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; @@ -1583,10 +1512,10 @@ struct llama_model_deepseek2ocr : public llama_model_base { using graph = llama_model_deepseek2::graph; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_glm_dsa : public llama_model_base { llama_model_glm_dsa(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1600,6 +1529,7 @@ struct llama_model_glm_dsa : public llama_model_base { graph_mtp(const llm_graph_params & params); }; + llama_rope_params get_rope_params_indexer(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; @@ -1618,7 +1548,6 @@ struct llama_model_eagle3 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_dflash : public llama_model_base { llama_model_dflash(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1648,7 +1577,6 @@ struct llama_model_dflash : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_mistral_large4 : public llama_model_base { llama_model_mistral_large4(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1658,10 +1586,10 @@ struct llama_model_mistral_large4 : public llama_model_base { graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params_indexer(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_mistral4 : public llama_model_deepseek2 { llama_model_mistral4(const struct llama_model_params & params) : llama_model_deepseek2(params) {} // reuse load_arch_hparams and load_arch_tensors from llama_model_deepseek2 @@ -1671,7 +1599,6 @@ struct llama_model_mistral4 : public llama_model_deepseek2 { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_chatglm : public llama_model_base { llama_model_chatglm(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1684,7 +1611,6 @@ struct llama_model_chatglm : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_glm4 : public llama_model_base { llama_model_glm4(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1697,7 +1623,6 @@ struct llama_model_glm4 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_glm4_moe : public llama_model_base { llama_model_glm4_moe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1714,7 +1639,6 @@ struct llama_model_glm4_moe : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_bitnet : public llama_model_base { llama_model_bitnet(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1727,7 +1651,6 @@ struct llama_model_bitnet : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_t5 : public llama_model_base { llama_model_t5(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1741,7 +1664,6 @@ struct llama_model_t5 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_t5encoder : public llama_model_base { llama_model_t5encoder(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1752,7 +1674,6 @@ struct llama_model_t5encoder : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_jais : public llama_model_base { llama_model_jais(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1765,7 +1686,6 @@ struct llama_model_jais : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_jais2 : public llama_model_base { llama_model_jais2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1778,7 +1698,6 @@ struct llama_model_jais2 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_nemotron : public llama_model_base { llama_model_nemotron(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1791,7 +1710,6 @@ struct llama_model_nemotron : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_nemotron_h : public llama_model_base { llama_model_nemotron_h(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1807,7 +1725,6 @@ struct llama_model_nemotron_h : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_nemotron_h_moe : public llama_model_nemotron_h { llama_model_nemotron_h_moe(const struct llama_model_params & params) : llama_model_nemotron_h(params) {} // reuse load_arch_hparams and load_arch_tensors from llama_model_nemotron_h @@ -1821,7 +1738,6 @@ struct llama_model_nemotron_h_moe : public llama_model_nemotron_h { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_metis16 : public llama_model_base { llama_model_metis16(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1854,7 +1770,6 @@ struct llama_model_metis16 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_exaone : public llama_model_base { llama_model_exaone(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1867,7 +1782,6 @@ struct llama_model_exaone : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_exaone4 : public llama_model_base { llama_model_exaone4(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1878,10 +1792,10 @@ struct llama_model_exaone4 : public llama_model_base { graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_exaone_moe : public llama_model_base { llama_model_exaone_moe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1891,10 +1805,10 @@ struct llama_model_exaone_moe : public llama_model_base { graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_rwkv6 : public llama_model_base { llama_model_rwkv6(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1907,7 +1821,6 @@ struct llama_model_rwkv6 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_rwkv6qwen2 : public llama_model_base { llama_model_rwkv6qwen2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1920,7 +1833,6 @@ struct llama_model_rwkv6qwen2 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_rwkv7 : public llama_model_base { llama_model_rwkv7(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1933,7 +1845,6 @@ struct llama_model_rwkv7 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_arwkv7 : public llama_model_base { llama_model_arwkv7(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1946,7 +1857,6 @@ struct llama_model_arwkv7 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_granite : public llama_model_base { llama_model_granite(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1972,7 +1882,6 @@ struct llama_model_granite : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_granite_moe : public llama_model_base { llama_model_granite_moe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1983,7 +1892,6 @@ struct llama_model_granite_moe : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_granite_switch : public llama_model_base { llama_model_granite_switch(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2031,7 +1939,6 @@ struct llama_model_granite_switch : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_minicpm : public llama_model_base { llama_model_minicpm(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2042,7 +1949,6 @@ struct llama_model_minicpm : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_granite_hybrid : public llama_model_base { llama_model_granite_hybrid(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2058,7 +1964,6 @@ struct llama_model_granite_hybrid : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_granite_swa : public llama_model_base { llama_model_granite_swa(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2081,10 +1986,10 @@ struct llama_model_granite_swa : public llama_model_base { const int il); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_chameleon : public llama_model_base { llama_model_chameleon(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2097,7 +2002,6 @@ struct llama_model_chameleon : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_wavtokenizer_dec : public llama_model_base { llama_model_wavtokenizer_dec(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2110,7 +2014,6 @@ struct llama_model_wavtokenizer_dec : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_plm : public llama_model_base { llama_model_plm(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2123,7 +2026,6 @@ struct llama_model_plm : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_limite : public llama_model_base { llama_model_limite(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2133,10 +2035,10 @@ struct llama_model_limite : public llama_model_base { graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_hrm_text : public llama_model_base { llama_model_hrm_text(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2155,7 +2057,6 @@ struct llama_model_hrm_text : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_bailingmoe : public llama_model_base { llama_model_bailingmoe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2168,7 +2069,6 @@ struct llama_model_bailingmoe : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_bailingmoe2 : public llama_model_base { llama_model_bailingmoe2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2181,7 +2081,6 @@ struct llama_model_bailingmoe2 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_bailingmoe3 : public llama_model_base { llama_model_bailingmoe3(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2203,7 +2102,6 @@ struct llama_model_bailingmoe3 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_seed_oss : public llama_model_base { llama_model_seed_oss(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2216,7 +2114,6 @@ struct llama_model_seed_oss : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_dots1 : public llama_model_base { llama_model_dots1(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2229,7 +2126,6 @@ struct llama_model_dots1 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_arcee : public llama_model_base { llama_model_arcee(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2242,7 +2138,6 @@ struct llama_model_arcee : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_afmoe : public llama_model_base { llama_model_afmoe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2255,7 +2150,6 @@ struct llama_model_afmoe : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_beam : public llama_model_base { llama_model_beam(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2290,10 +2184,10 @@ struct llama_model_laguna : public llama_model_base { graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_ernie4_5 : public llama_model_base { llama_model_ernie4_5(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2306,7 +2200,6 @@ struct llama_model_ernie4_5 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_ernie4_5_moe : public llama_model_ernie4_5 { llama_model_ernie4_5_moe(const struct llama_model_params & params) : llama_model_ernie4_5(params) {} // reuse load_arch_hparams and load_arch_tensors from llama_model_ernie4_5 @@ -2318,7 +2211,6 @@ struct llama_model_ernie4_5_moe : public llama_model_ernie4_5 { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_paddleocr : public llama_model_ernie4_5 { llama_model_paddleocr(const struct llama_model_params & params) : llama_model_ernie4_5(params) {} // reuse load_arch_hparams and load_arch_tensors from llama_model_ernie4_5 @@ -2330,7 +2222,6 @@ struct llama_model_paddleocr : public llama_model_ernie4_5 { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_hunyuan_moe : public llama_model_base { llama_model_hunyuan_moe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2340,6 +2231,7 @@ struct llama_model_hunyuan_moe : public llama_model_base { graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; @@ -2359,7 +2251,6 @@ struct llama_model_hy_v3 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_hy_v4 : public llama_model_base { llama_model_hy_v4(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2426,10 +2317,10 @@ struct llama_model_hy_v4 : public llama_model_base { int il) const; }; + llama_rope_params get_rope_params_indexer(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_hunyuan_vl : public llama_model_base { llama_model_hunyuan_vl(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2442,17 +2333,16 @@ struct llama_model_hunyuan_vl : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_hunyuan_dense : public llama_model_hunyuan_vl { llama_model_hunyuan_dense(const struct llama_model_params & params) : llama_model_hunyuan_vl(params) {} // reuse load_arch_hparams and load_arch_tensors from llama_model_hunyuan_vl using graph = llama_model_hunyuan_vl::graph; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_smollm3 : public llama_model_base { llama_model_smollm3(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2465,7 +2355,6 @@ struct llama_model_smollm3 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_openai_moe : public llama_model_base { llama_model_openai_moe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2478,7 +2367,6 @@ struct llama_model_openai_moe : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_falcon_h1 : public llama_model_base { llama_model_falcon_h1(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2488,10 +2376,10 @@ struct llama_model_falcon_h1 : public llama_model_base { graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_lfm2 : public llama_model_base { llama_model_lfm2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2512,10 +2400,10 @@ struct llama_model_lfm2 : public llama_model_base { ggml_tensor * inp_out_ids); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_lfm2moe : public llama_model_base { llama_model_lfm2moe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2527,7 +2415,6 @@ struct llama_model_lfm2moe : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_lfm3 : public llama_model_base { llama_model_lfm3(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2542,7 +2429,6 @@ struct llama_model_lfm3 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_smallthinker : public llama_model_base { llama_model_smallthinker(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2556,7 +2442,6 @@ struct llama_model_smallthinker : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_grovemoe : public llama_model_base { llama_model_grovemoe(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2569,7 +2454,6 @@ struct llama_model_grovemoe : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_apertus : public llama_model_base { llama_model_apertus(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2582,7 +2466,6 @@ struct llama_model_apertus : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_minimax_01 : public llama_model_base { llama_model_minimax_01(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2595,7 +2478,6 @@ struct llama_model_minimax_01 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_minimax_m2 : public llama_model_base { llama_model_minimax_m2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2641,10 +2523,10 @@ struct llama_model_cogvlm : public llama_model_base { graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_pangu_embed : public llama_model_base { llama_model_pangu_embed(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2654,10 +2536,10 @@ struct llama_model_pangu_embed : public llama_model_base { graph(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_qwen3next : public llama_model_base { llama_model_qwen3next(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2700,7 +2582,6 @@ struct llama_model_qwen3next : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_qwen35 : public llama_model_base { llama_model_qwen35(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -2713,7 +2594,6 @@ struct llama_model_qwen35 : public llama_model_base { llm_graph_input_attn_kv * inp_attn, ggml_tensor * cur, ggml_tensor * inp_pos, - int * sections, int il); ggml_tensor * build_layer_attn_linear( @@ -2744,7 +2624,6 @@ struct llama_model_qwen35 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - // Qwen3.5 backbone evaluated without memory, with a joint decision head on top of its hidden states struct llama_model_clef : public llama_model_qwen35 { llama_model_clef(const struct llama_model_params & params) : llama_model_qwen35(params) {} @@ -2815,7 +2694,6 @@ struct llama_model_clef : public llama_model_qwen35 { llm_graph_input_attn_no_cache * inp_attn, ggml_tensor * cur, ggml_tensor * inp_pos, - int * sections, int il); ggml_tensor * build_layer_attn_linear( @@ -2837,7 +2715,6 @@ struct llama_model_clef : public llama_model_qwen35 { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_qwen4exp : public llama_model_base { llama_model_qwen4exp(const struct llama_model_params & params) : llama_model_base(params) {} @@ -2881,7 +2758,6 @@ struct llama_model_qwen4exp : public llama_model_base { llm_graph_input_kpool * inp_kpool, ggml_tensor * cur, ggml_tensor * inp_pos, - int * sections, int il); // dense self-attention over the cells the QSA mask keeps @@ -2904,7 +2780,6 @@ struct llama_model_qwen4exp : public llama_model_base { llm_graph_input_kpool * inp_kpool, ggml_tensor * cur, ggml_tensor * inp_pos, - int * sections, int il); ggml_tensor * build_layer_attn_linear( @@ -2969,7 +2844,6 @@ struct llama_model_qwen35moe : public llama_model_base { llm_graph_input_attn_kv * inp_attn, ggml_tensor * cur, ggml_tensor * inp_pos, - int * sections, int il); ggml_tensor * build_layer_attn_linear( @@ -3000,7 +2874,6 @@ struct llama_model_qwen35moe : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_clover_moe : public llama_model_base { llama_model_clover_moe(const llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -3033,7 +2906,6 @@ struct llama_model_mistral3 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_mimo2 : public llama_model_base { llama_model_mimo2(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -3050,7 +2922,6 @@ struct llama_model_mimo2 : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_kimi_k3 : public llama_model_base { llama_model_kimi_k3(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -3207,7 +3078,6 @@ struct llama_model_kimi_linear : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_step35 : public llama_model_base { llama_model_step35(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -3221,10 +3091,10 @@ struct llama_model_step35 : public llama_model_base { graph_mtp(const llm_graph_params & params); }; + llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_spark2_5 : public llama_model_base { llama_model_spark2_5(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -3251,7 +3121,6 @@ struct llama_model_k2_horizon : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; - struct llama_model_kolibri1 : public llama_model_base { llama_model_kolibri1(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; diff --git a/src/models/modern-bert.cpp b/src/models/modern-bert.cpp index 5c6718897..17c9c5ab6 100644 --- a/src/models/modern-bert.cpp +++ b/src/models/modern-bert.cpp @@ -160,8 +160,6 @@ llama_model_modern_bert::graph::graph(const llm_graph_params & params) : llm_gra const int n_layer_enc = n_layer - n_layer_dec; for (int il = 0; il < n_layer_enc; ++il) { - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); cur = inpL; @@ -178,17 +176,8 @@ llama_model_modern_bert::graph::graph(const llm_graph_params & params) : llm_gra n_embd_head, n_head, n_head_kv, il); // RoPE - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/muse-glimmer.cpp b/src/models/muse-glimmer.cpp index 768010c87..e3f478fe3 100644 --- a/src/models/muse-glimmer.cpp +++ b/src/models/muse-glimmer.cpp @@ -11,6 +11,7 @@ void llama_model_muse_glimmer::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; load_swa_pattern(ml, 4); + hparams.rope_pattern = hparams.is_swa_impl; switch (hparams.n_layer()) { case 52: type = LLM_TYPE_30B; break; @@ -79,14 +80,8 @@ llama_model_muse_glimmer::graph::graph(const llm_graph_params & params) // expose per-layer residual for speculative drafts (see LLM_KV_TARGET_LAYERS). res->t_layer_inp[il] = inpL; - const float freq_base_l = model.get_rope_freq_base (cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - ggml_tensor * inpSA = inpL; - // RoPE runs on the SWA layers, NoPE on full ones. - const bool use_rope = hparams.is_swa(il); - // pre-attention norm (weight+1 folded at conversion time) cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); @@ -109,19 +104,11 @@ llama_model_muse_glimmer::graph::graph(const llm_graph_params & params) cb(Qcur, "Qcur_normed", il); cb(Kcur, "Kcur_normed", il); - if (use_rope) { - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Qcur, "Qcur_rope", il); + Qcur = build_rope(Qcur, inp_pos, il); + cb(Qcur, "Qcur_rope", il); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Kcur, "Kcur_rope", il); - } + Kcur = build_rope(Kcur, inp_pos, il); + cb(Kcur, "Kcur_rope", il); // SDPA. wo is deferred; the gate goes between attn_out and o_proj. cur = build_attn(inp_attn, diff --git a/src/models/muse-spark.cpp b/src/models/muse-spark.cpp index 30dd2d1ca..ea6ab84ab 100644 --- a/src/models/muse-spark.cpp +++ b/src/models/muse-spark.cpp @@ -15,6 +15,7 @@ void llama_model_muse_spark::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); + hparams.rope_pattern = hparams.is_swa_impl; switch (hparams.n_layer()) { case 62: type = LLM_TYPE_816B_A42B; break; @@ -96,14 +97,8 @@ llama_model_muse_spark::graph::graph(const llm_graph_params & params) for (int il = 0; il < n_layer; ++il) { res->t_layer_inp[il] = inpL; - const float freq_base_l = model.get_rope_freq_base (cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - ggml_tensor * inpSA = inpL; - // RoPE runs on the SWA layers, NoPE on the global ones. - const bool use_rope = hparams.is_swa(il); - cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); @@ -120,19 +115,11 @@ llama_model_muse_spark::graph::graph(const llm_graph_params & params) cb(Qcur, "Qcur_normed", il); cb(Kcur, "Kcur_normed", il); - if (use_rope) { - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Qcur, "Qcur_rope", il); + Qcur = build_rope(Qcur, inp_pos, il); + cb(Qcur, "Qcur_rope", il); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(Kcur, "Kcur_rope", il); - } + Kcur = build_rope(Kcur, inp_pos, il); + cb(Kcur, "Kcur_rope", il); cur = build_attn(inp_attn, NULL, NULL, NULL, diff --git a/src/models/nanbeige.cpp b/src/models/nanbeige.cpp index 43235b86f..2f3e0ff1c 100644 --- a/src/models/nanbeige.cpp +++ b/src/models/nanbeige.cpp @@ -110,20 +110,12 @@ llama_model_nanbeige::graph::graph(const llm_graph_params & params) : cb(cur, "attn_norm", il); { - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/nemotron.cpp b/src/models/nemotron.cpp index a6126c652..f769b2537 100644 --- a/src/models/nemotron.cpp +++ b/src/models/nemotron.cpp @@ -84,17 +84,8 @@ llama_model_nemotron::graph::graph(const llm_graph_params & params) : llm_graph_ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/neo-bert.cpp b/src/models/neo-bert.cpp index 21bd380ee..58a012a68 100644 --- a/src/models/neo-bert.cpp +++ b/src/models/neo-bert.cpp @@ -70,17 +70,8 @@ llama_model_neo_bert::graph::graph(const llm_graph_params & params) : llm_graph_ n_embd_head, n_head, n_head_kv, il); // RoPE - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/olmo.cpp b/src/models/olmo.cpp index 710cef98e..2afead631 100644 --- a/src/models/olmo.cpp +++ b/src/models/olmo.cpp @@ -73,17 +73,8 @@ llama_model_olmo::graph::graph(const llm_graph_params & params) : llm_graph_cont auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/olmo2.cpp b/src/models/olmo2.cpp index ff09b07dd..c0a3e8295 100644 --- a/src/models/olmo2.cpp +++ b/src/models/olmo2.cpp @@ -51,6 +51,21 @@ void llama_model_olmo2::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_olmo2::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + res.freq_base = cparams.rope_freq_base; + + // regular RoPE on the SWA layers, without YaRN + if (hparams.is_swa(il)) { + res.freq_scale = 1.0f; + res.ext_factor = 0.0f; + res.attn_factor = 1.0f; + } + + return res; +} + std::unique_ptr llama_model_olmo2::build_arch_graph(const llm_graph_params & params) const { if (hparams.swa_type == LLAMA_SWA_TYPE_STANDARD) { return std::make_unique>(params); @@ -112,36 +127,8 @@ llama_model_olmo2::graph::graph(const llm_graph_params & params) : llm_gra Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); - const bool is_swa = hparams.is_swa(il); - - if (is_swa) { - // For sliding window layers, Olmo3 use regular rope with no yarn rope scaling. - // This is achieved here by setting freq_scale and attn_factor to 1. - // We also set ext_factor to 0 to avoid a few unnecessary computations. - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, 1.0, - 0.0, 1.0, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, 1.0, - 0.0, 1.0, beta_fast, beta_slow - ); - } else { - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); diff --git a/src/models/olmoe.cpp b/src/models/olmoe.cpp index 88c5d2115..9ebdd9c94 100644 --- a/src/models/olmoe.cpp +++ b/src/models/olmoe.cpp @@ -100,17 +100,8 @@ llama_model_olmoe::graph::graph(const llm_graph_params & params) : llm_graph_con Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/openai-moe.cpp b/src/models/openai-moe.cpp index a05ad1cb5..920acbb64 100644 --- a/src/models/openai-moe.cpp +++ b/src/models/openai-moe.cpp @@ -75,9 +75,6 @@ llama_model_openai_moe::graph::graph(const llm_graph_params & params) : llm_grap for (int il = 0; il < n_layer; ++il) { res->t_layer_inp[il] = inpL; - const float freq_base_l = model.get_rope_freq_base (cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - ggml_tensor * inpSA = inpL; // norm @@ -92,17 +89,8 @@ llama_model_openai_moe::graph::graph(const llm_graph_params & params) : llm_grap auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_rot, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/openelm.cpp b/src/models/openelm.cpp index e9b43c749..3328bd54c 100644 --- a/src/models/openelm.cpp +++ b/src/models/openelm.cpp @@ -103,17 +103,8 @@ llama_model_openelm::graph::graph(const llm_graph_params & params) : llm_graph_c LLM_NORM_RMS, il); cb(Kcur, "Kcur", il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, NULL, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, NULL, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/orion.cpp b/src/models/orion.cpp index 2c6e361ff..4fe921189 100644 --- a/src/models/orion.cpp +++ b/src/models/orion.cpp @@ -73,17 +73,8 @@ llama_model_orion::graph::graph(const llm_graph_params & params) : llm_graph_con auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/paddleocr.cpp b/src/models/paddleocr.cpp index 084536784..4dbd9e215 100644 --- a/src/models/paddleocr.cpp +++ b/src/models/paddleocr.cpp @@ -19,9 +19,6 @@ llama_model_paddleocr::graph::graph(const llm_graph_params & params) : inpL = build_inp_embd(model.tok_embd); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); @@ -42,17 +39,8 @@ llama_model_paddleocr::graph::graph(const llm_graph_params & params) : auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_multi( - ctx0, Qcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_multi( - ctx0, Kcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/pangu-embed.cpp b/src/models/pangu-embed.cpp index d8d0a146d..336c71a41 100644 --- a/src/models/pangu-embed.cpp +++ b/src/models/pangu-embed.cpp @@ -51,6 +51,14 @@ void llama_model_pangu_embed::load_arch_tensors(llama_model_loader &) { } } +llama_rope_params llama_model_pangu_embed::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + res.factors = nullptr; + + return res; +} + std::unique_ptr llama_model_pangu_embed::build_arch_graph(const llm_graph_params & params) const { return std::make_unique(params); } @@ -88,16 +96,8 @@ llama_model_pangu_embed::graph::graph(const llm_graph_params & params) : llm_gra auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/phi2.cpp b/src/models/phi2.cpp index d92dfc6e0..90c502cfd 100644 --- a/src/models/phi2.cpp +++ b/src/models/phi2.cpp @@ -74,17 +74,8 @@ llama_model_phi2::graph::graph(const llm_graph_params & params) : llm_graph_cont { auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], attn_norm_output, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/phi3.cpp b/src/models/phi3.cpp index c7a1d2131..815ab0465 100644 --- a/src/models/phi3.cpp +++ b/src/models/phi3.cpp @@ -93,8 +93,6 @@ llama_model_phi3::graph::graph(const llm_graph_params & params) : llm_grap // self-attention { - // rope freq factors for 128k context - ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); ggml_tensor* attn_norm_output = build_norm(inpL, model.layers[il].attn_norm, @@ -104,17 +102,8 @@ llama_model_phi3::graph::graph(const llm_graph_params & params) : llm_grap auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], attn_norm_output, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/plamo.cpp b/src/models/plamo.cpp index e48ff2bb0..10348f291 100644 --- a/src/models/plamo.cpp +++ b/src/models/plamo.cpp @@ -69,17 +69,8 @@ llama_model_plamo::graph::graph(const llm_graph_params & params) : llm_graph_con auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_embd_head, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_embd_head, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/plamo2.cpp b/src/models/plamo2.cpp index d392c3fb4..2d9edf56e 100644 --- a/src/models/plamo2.cpp +++ b/src/models/plamo2.cpp @@ -233,14 +233,12 @@ ggml_tensor * llama_model_plamo2::graph::build_plamo2_attn_layer(llm_graph_input Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = build_rope(Kcur, inp_pos, il); cur = build_attn(inp, model.layers[il].wo, NULL, model.layers[il].wo_s, diff --git a/src/models/plamo3.cpp b/src/models/plamo3.cpp index ab07450a3..d6de1a356 100644 --- a/src/models/plamo3.cpp +++ b/src/models/plamo3.cpp @@ -89,16 +89,6 @@ llama_model_plamo3::graph::graph(const llm_graph_params & params) : for (int il = 0; il < n_layer; ++il) { ggml_tensor * residual = inpL; - float freq_base_l = 0.0f; - float freq_scale_l = 0.0f; - if constexpr (iswa) { - freq_base_l = model.get_rope_freq_base (cparams, il); - freq_scale_l = model.get_rope_freq_scale(cparams, il); - } else { - freq_base_l = freq_base; - freq_scale_l = freq_scale; - } - cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); @@ -128,12 +118,8 @@ llama_model_plamo3::graph::graph(const llm_graph_params & params) : Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "attn_k_norm", il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); const float attn_scale = 1.0f / sqrtf(float(head_dim_q)); diff --git a/src/models/plm.cpp b/src/models/plm.cpp index 61b530e8b..b0069e49b 100644 --- a/src/models/plm.cpp +++ b/src/models/plm.cpp @@ -133,20 +133,12 @@ llama_model_plm::graph::graph(const llm_graph_params & params) : llm_graph_conte 0); cb(v_states, "v_states", il); - q = ggml_rope_ext( - ctx0, q, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + q = build_rope(q, inp_pos, il); q = ggml_rope_set_offset(q, n_embd_head_qk_nope); cb(q, "q_rope", il); // shared RoPE key - k_pe = ggml_rope_ext( - ctx0, k_pe, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "k_pe", il); ggml_tensor * q_states = q; diff --git a/src/models/pockettts.cpp b/src/models/pockettts.cpp index a55fe138f..a7dcb5366 100644 --- a/src/models/pockettts.cpp +++ b/src/models/pockettts.cpp @@ -74,17 +74,8 @@ llama_model_pockettts::graph::graph(const llm_graph_params & params) : llm_graph auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/qwen.cpp b/src/models/qwen.cpp index e009069db..4e0b1986d 100644 --- a/src/models/qwen.cpp +++ b/src/models/qwen.cpp @@ -71,17 +71,8 @@ llama_model_qwen::graph::graph(const llm_graph_params & params) : llm_graph_cont n_embd_head, n_head, n_head_kv, il); // using mode = 2 for neox mode - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/qwen2.cpp b/src/models/qwen2.cpp index f38e7a0a3..bb93ca9b1 100644 --- a/src/models/qwen2.cpp +++ b/src/models/qwen2.cpp @@ -83,17 +83,8 @@ llama_model_qwen2::graph::graph(const llm_graph_params & params) : llm_graph_con auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/qwen2moe.cpp b/src/models/qwen2moe.cpp index 6d898eefd..e24919bc6 100644 --- a/src/models/qwen2moe.cpp +++ b/src/models/qwen2moe.cpp @@ -95,17 +95,8 @@ llama_model_qwen2moe::graph::graph(const llm_graph_params & params) : llm_graph_ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/qwen2vl.cpp b/src/models/qwen2vl.cpp index d87c14ba4..5831fc435 100644 --- a/src/models/qwen2vl.cpp +++ b/src/models/qwen2vl.cpp @@ -55,9 +55,6 @@ llama_model_qwen2vl::graph::graph(const llm_graph_params & params) : llm_graph_c auto * inp_attn = build_attn_inp_kv(); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - ggml_tensor * inp_out_ids = build_inp_out_ids(); for (int il = 0; il < n_layer; ++il) { @@ -75,17 +72,8 @@ llama_model_qwen2vl::graph::graph(const llm_graph_params & params) : llm_graph_c auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_multi( - ctx0, Qcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_multi( - ctx0, Kcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/qwen3.cpp b/src/models/qwen3.cpp index 28d01db54..7a6052970 100644 --- a/src/models/qwen3.cpp +++ b/src/models/qwen3.cpp @@ -102,20 +102,12 @@ ggml_tensor * llama_model_qwen3::graph::build_backbone() { Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index 5601d2052..d0b4f4c2c 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -140,9 +140,6 @@ llama_model_qwen35::graph::graph(const llm_graph_params & params) : GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - ggml_tensor * cur; ggml_tensor * inpL; @@ -172,7 +169,7 @@ llama_model_qwen35::graph::graph(const llm_graph_params & params) : cur = build_layer_attn_linear(inp->get_recr(), cur, il); } else { // Full attention layer - cur = build_layer_attn(inp->get_attn(), cur, inp_pos, sections, il); + cur = build_layer_attn(inp->get_attn(), cur, inp_pos, il); } if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) { @@ -267,7 +264,6 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn( llm_graph_input_attn_kv * inp, ggml_tensor * cur, ggml_tensor * inp_pos, - int * sections, int il) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -308,17 +304,8 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn( Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); // Apply MRoPE - Qcur = ggml_rope_multi( - ctx0, Qcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_multi( - ctx0, Kcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); @@ -509,9 +496,6 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llm_graph_params & params) GGML_ASSERT(layer.nextn.enorm && "MTP block missing nextn.enorm"); GGML_ASSERT(layer.nextn.hnorm && "MTP block missing nextn.hnorm"); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - // TODO: extract in a common llm_graph_context::build_inp_embd_h() auto inp = std::make_unique(hparams.n_embd_inp(), hparams.n_embd); @@ -604,12 +588,8 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llm_graph_params & params) Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); cb(Vcur, "mtp_Vcur", il); - Qcur = ggml_rope_multi(ctx0, Qcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale; diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cpp index eb23efdd3..0b06b1f5d 100644 --- a/src/models/qwen35moe.cpp +++ b/src/models/qwen35moe.cpp @@ -159,9 +159,6 @@ llama_model_qwen35moe::graph::graph(const llm_graph_params & params) : GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - ggml_tensor * cur; ggml_tensor * inpL; @@ -191,7 +188,7 @@ llama_model_qwen35moe::graph::graph(const llm_graph_params & params) : cur = build_layer_attn_linear(inp->get_recr(), cur, il); } else { // Full attention layer - cur = build_layer_attn(inp->get_attn(), cur, inp_pos, sections, il); + cur = build_layer_attn(inp->get_attn(), cur, inp_pos, il); } if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) { @@ -277,7 +274,6 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn( llm_graph_input_attn_kv * inp, ggml_tensor * cur, ggml_tensor * inp_pos, - int * sections, int il) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -318,17 +314,8 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn( Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); // Apply IMRoPE - Qcur = ggml_rope_multi( - ctx0, Qcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_multi( - ctx0, Kcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); @@ -558,9 +545,6 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llm_graph_params & params) GGML_ASSERT(layer.nextn.hnorm && "MTP block missing nextn.hnorm"); GGML_ASSERT(layer.ffn_gate_inp && "MTP block missing ffn_gate_inp"); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - // TODO: extract in a common llm_graph_context::build_inp_embd_h() auto inp = std::make_unique(hparams.n_embd_inp(), hparams.n_embd); @@ -643,12 +627,8 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llm_graph_params & params) Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); cb(Vcur, "mtp_Vcur", il); - Qcur = ggml_rope_multi(ctx0, Qcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale; diff --git a/src/models/qwen3moe.cpp b/src/models/qwen3moe.cpp index 99263096b..ce7144a22 100644 --- a/src/models/qwen3moe.cpp +++ b/src/models/qwen3moe.cpp @@ -97,20 +97,12 @@ llama_model_qwen3moe::graph::graph(const llm_graph_params & params) : llm_graph_ Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/qwen3next.cpp b/src/models/qwen3next.cpp index 864cb0343..0731838f8 100644 --- a/src/models/qwen3next.cpp +++ b/src/models/qwen3next.cpp @@ -273,15 +273,8 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_attn( Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, - freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); @@ -705,12 +698,8 @@ llama_model_qwen3next::graph_mtp::graph_mtp(const llm_graph_params & params) Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "mtp_Qcur", il); cb(Kcur, "mtp_Kcur", il); diff --git a/src/models/qwen3vl.cpp b/src/models/qwen3vl.cpp index 7c07e6c94..480e570dc 100644 --- a/src/models/qwen3vl.cpp +++ b/src/models/qwen3vl.cpp @@ -71,9 +71,6 @@ llama_model_qwen3vl::graph::graph(const llm_graph_params & params) : llm_graph_c inpL = build_inp_embd(model.tok_embd); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); @@ -99,20 +96,12 @@ llama_model_qwen3vl::graph::graph(const llm_graph_params & params) : llm_graph_c Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_multi( - ctx0, Qcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Kcur = ggml_rope_multi( - ctx0, Kcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/qwen3vlmoe.cpp b/src/models/qwen3vlmoe.cpp index 20835cb84..c13f61990 100644 --- a/src/models/qwen3vlmoe.cpp +++ b/src/models/qwen3vlmoe.cpp @@ -75,9 +75,6 @@ llama_model_qwen3vlmoe::graph::graph(const llm_graph_params & params) : llm_grap inpL = build_inp_embd(model.tok_embd); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - // inp_pos - contains the positions ggml_tensor * inp_pos = build_inp_pos(); @@ -103,20 +100,12 @@ llama_model_qwen3vlmoe::graph::graph(const llm_graph_params & params) : llm_grap Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_multi( - ctx0, Qcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Kcur = ggml_rope_multi( - ctx0, Kcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index a81d62662..c543edf0a 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -450,9 +450,6 @@ llama_model_qwen4exp::graph::graph(const llm_graph_params & params) : GGML_ASSERT(hparams.n_embd_head_v() == hparams.n_embd_head_k()); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - auto * inp = build_inp_mem_hybrid(); // qwen4exp always builds llama_memory_hybrid_idx, so this downcast is safe @@ -519,7 +516,7 @@ llama_model_qwen4exp::graph::graph(const llm_graph_params & params) : if (hparams.is_recr(il)) { cur = build_layer_attn_linear(inp->get_recr(), cur, il); } else { - cur = build_layer_attn(inp->get_attn(), mctx_hyb, inp_kpool, cur, inp_pos, sections, il); + cur = build_layer_attn(inp->get_attn(), mctx_hyb, inp_kpool, cur, inp_pos, il); } if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) { @@ -591,9 +588,6 @@ llama_model_qwen4exp::graph_mtp::graph_mtp(const llm_graph_params & params) : GGML_ASSERT(layer.nextn.eh_proj && layer.nextn.enorm && layer.nextn.hnorm && layer.nextn.hc_head_norm && "MTP block missing, load the model with MTP enabled"); - int sections[4]; - std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); - auto inp = std::make_unique(hparams.n_embd_inp(), hparams.n_embd_out()); inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); @@ -646,7 +640,7 @@ llama_model_qwen4exp::graph_mtp::graph_mtp(const llm_graph_params & params) : hc_mix attn_mix = build_hc_mix(res_hc, layer.hc_attn_norm, layer.hc_attn_down, layer.hc_attn_up, layer.hc_attn_inject, layer.hc_attn_down_inject, il); ggml_tensor * cur = attn_mix.cur; - cur = build_layer_attn(inp_hyb->get_attn(), mctx_hyb, inp_kpool, cur, inp_pos, sections, il); + cur = build_layer_attn(inp_hyb->get_attn(), mctx_hyb, inp_kpool, cur, inp_pos, il); res_hc = build_hc_combine(res_hc, cur, attn_mix.inject, il); hc_mix ffn_mix = build_hc_mix(res_hc, layer.hc_ffn_norm, layer.hc_ffn_down, layer.hc_ffn_up, layer.hc_ffn_inject, layer.hc_ffn_down_inject, il); @@ -818,7 +812,6 @@ ggml_tensor * llama_model_qwen4exp::graph::build_qsa_sel( llm_graph_input_kpool * inp_kpool, ggml_tensor * cur, ggml_tensor * inp_pos, - int * sections, int il) { const llama_kv_cache_context * mctx_idx = mctx_hyb->get_idx(); @@ -855,9 +848,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_qsa_sel( pooled_new = build_norm(pooled_new, model.layers[il].index_k_norm, nullptr, LLM_NORM_RMS, il); pooled_new = ggml_reshape_3d(ctx0, pooled_new, idx_dim, 1, n_new); - pooled_new = ggml_rope_multi(ctx0, pooled_new, inp_kpool->new_pool_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + pooled_new = build_rope(pooled_new, inp_kpool->new_pool_pos, il); pooled_new = ggml_reshape_2d(ctx0, pooled_new, idx_dim, n_new); cb(pooled_new, "indexer_pool_k_new", il); @@ -871,9 +862,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_qsa_sel( ggml_tensor * q = build_lora_mm(model.layers[il].index_q_proj, cur); q = ggml_reshape_3d(ctx0, q, idx_dim, n_idx_h, n_tokens); q = build_norm(q, model.layers[il].index_q_norm, nullptr, LLM_NORM_RMS, il); - q = ggml_rope_multi(ctx0, q, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + q = build_rope(q, inp_pos, il); cb(q, "indexer_q", il); // the reference sums the rectified head scores unweighted, scaled by 1/sqrt(head_dim), @@ -1022,7 +1011,6 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn( llm_graph_input_kpool * inp_kpool, ggml_tensor * cur, ggml_tensor * inp_pos, - int * sections, int il) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -1030,7 +1018,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn( // indexer reads the same block input as q/k/v; no cache or no ratio means dense const bool qsa = inp_kpool != nullptr && hparams.dsv4_compress_ratios[il] > 0; - ggml_tensor * sel = qsa ? build_qsa_sel(mctx_hyb, inp_kpool, cur, inp_pos, sections, il) : nullptr; + ggml_tensor * sel = qsa ? build_qsa_sel(mctx_hyb, inp_kpool, cur, inp_pos, il) : nullptr; // Qwen3Next uses a single Q projection that outputs query + gate ggml_tensor * Qcur_full = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s); // [ (n_embd_head * 2) * n_head, n_tokens ] @@ -1064,17 +1052,8 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn( Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); // Apply IMRoPE - Qcur = ggml_rope_multi( - ctx0, Qcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_multi( - ctx0, Kcur, inp_pos, nullptr, - n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/ricursive.cpp b/src/models/ricursive.cpp index cd5dc3547..e14caa8b3 100644 --- a/src/models/ricursive.cpp +++ b/src/models/ricursive.cpp @@ -162,17 +162,8 @@ llama_model_ricursive::graph::graph(const llm_graph_params & params) : llm_graph cb(Kcur, "Kcur_normed", il); // global layers use no RoPE (NoPE): the reference rotates only sliding_attention layers (modeling_ricursive.py:127-129,203-205) - if (hparams.has_rope(il)) { - const float freq_base_l = model.get_rope_freq_base (cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); diff --git a/src/models/rnd1.cpp b/src/models/rnd1.cpp index f55448433..8ea8d2364 100644 --- a/src/models/rnd1.cpp +++ b/src/models/rnd1.cpp @@ -99,20 +99,12 @@ llama_model_rnd1::graph::graph(const llm_graph_params & params) : llm_graph_cont Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); cb(Kcur, "Kcur_normed", il); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/seed-oss.cpp b/src/models/seed-oss.cpp index fcf66290b..c7940dc80 100644 --- a/src/models/seed-oss.cpp +++ b/src/models/seed-oss.cpp @@ -81,17 +81,8 @@ llama_model_seed_oss::graph::graph(const llm_graph_params & params) : llm_graph_ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/smallthinker.cpp b/src/models/smallthinker.cpp index 0e9e36ee1..2c183e33c 100644 --- a/src/models/smallthinker.cpp +++ b/src/models/smallthinker.cpp @@ -16,6 +16,10 @@ void llama_model_smallthinker::load_arch_hparams(llama_model_loader & ml) { hparams.n_no_rope_layer_step = hparams.n_layer(); } + for (uint32_t il = 0; il < hparams.n_layer_all; ++il) { + hparams.rope_pattern[il] = hparams.n_no_rope_layer_step == hparams.n_layer() || il % hparams.n_no_rope_layer_step != 0; + } + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false); @@ -97,15 +101,9 @@ llama_model_smallthinker::graph::graph(const llm_graph_params & params) : ggml_tensor * inp_out_ids = build_inp_out_ids(); for (int il = 0; il < n_layer; ++il) { - const float freq_base_l = model.get_rope_freq_base (cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); ggml_tensor * inpSA = inpL; - // This overlaps with SWA layers in current models, so get_rope_freq_base/scale may be superfluous - const bool use_rope = hparams.n_no_rope_layer_step == n_layer || - il % hparams.n_no_rope_layer_step != 0; - ggml_tensor * probs = build_lora_mm(model.layers[il].ffn_gate_inp, inpL); // [n_expert, n_tokens] cb(probs, "ffn_moe_logits", il); @@ -119,13 +117,8 @@ llama_model_smallthinker::graph::graph(const llm_graph_params & params) : auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - if (use_rope) { - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/smollm3.cpp b/src/models/smollm3.cpp index 438e5d334..fa9e9c539 100644 --- a/src/models/smollm3.cpp +++ b/src/models/smollm3.cpp @@ -3,6 +3,9 @@ void llama_model_smollm3::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); hparams.n_no_rope_layer_step = 4; + for (uint32_t il = 0; il < hparams.n_layer_all; ++il) { + hparams.rope_pattern[il] = (il + 1) % hparams.n_no_rope_layer_step != 0; + } switch (hparams.n_layer()) { case 36: type = LLM_TYPE_3B; break; @@ -66,8 +69,6 @@ llama_model_smollm3::graph::graph(const llm_graph_params & params) : llm_graph_c for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; - const bool use_rope = (il + 1) % hparams.n_no_rope_layer_step != 0; - // norm cur = build_norm(inpL, model.layers[il].attn_norm, NULL, @@ -80,19 +81,8 @@ llama_model_smollm3::graph::graph(const llm_graph_params & params) : llm_graph_c auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - if (use_rope) { - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - } + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); diff --git a/src/models/spark2-5.cpp b/src/models/spark2-5.cpp index 1c6543a2c..e90184d8e 100644 --- a/src/models/spark2-5.cpp +++ b/src/models/spark2-5.cpp @@ -73,19 +73,12 @@ llama_model_spark2_5::graph::graph(const llm_graph_params & params) : llm_graph_ const int64_t n_head_i = hparams.n_head(il); const int64_t n_head_kv_i = hparams.n_head_kv(il); - const int64_t n_rot_i = hparams.n_rot(il); - const float freq_base_i = model.get_rope_freq_base(cparams, il); - const float freq_scale_i = model.get_rope_freq_scale(cparams, il); ggml_tensor * attn_inp = cur; auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head_i, n_head_kv_i, il); - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, - n_rot_i, rope_type, n_ctx_orig, freq_base_i, freq_scale_i, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, - n_rot_i, rope_type, n_ctx_orig, freq_base_i, freq_scale_i, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur_rope", il); cb(Kcur, "Kcur_rope", il); diff --git a/src/models/stablelm.cpp b/src/models/stablelm.cpp index 84e9b8c76..5aec7beee 100644 --- a/src/models/stablelm.cpp +++ b/src/models/stablelm.cpp @@ -96,17 +96,8 @@ llama_model_stablelm::graph::graph(const llm_graph_params & params) : llm_graph_ cb(Kcur, "Kcur", il); } - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/starcoder2.cpp b/src/models/starcoder2.cpp index 54f9cb8b1..2b670d562 100644 --- a/src/models/starcoder2.cpp +++ b/src/models/starcoder2.cpp @@ -89,17 +89,8 @@ llama_model_starcoder2::graph::graph(const llm_graph_params & params) : llm_grap auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/src/models/step35.cpp b/src/models/step35.cpp index bd06ce0db..ad8a3a1c7 100644 --- a/src/models/step35.cpp +++ b/src/models/step35.cpp @@ -172,6 +172,16 @@ void llama_model_step35::load_arch_tensors(llama_model_loader & ml) { } } +llama_rope_params llama_model_step35::get_rope_params(const llama_cparams & cparams, int il) const { + auto res = llama_model::get_rope_params(cparams, il); + + if (hparams.is_swa(il)) { + res.factors = nullptr; + } + + return res; +} + std::unique_ptr llama_model_step35::build_arch_graph(const llm_graph_params & params) const { if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { return std::make_unique(params); @@ -195,9 +205,6 @@ llama_model_step35::graph::graph(const llm_graph_params & params) : llm_graph_co const uint32_t n_head_l = hparams.n_head(il); const uint32_t n_head_kv_l = hparams.n_head_kv(il); - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - cur = inpL; // dump pre-attn RMSNorm input to pinpoint layer boundary issues @@ -232,19 +239,8 @@ llama_model_step35::graph::graph(const llm_graph_params & params) : llm_graph_co } // RoPE (partial rotary factors per layer) - const bool is_swa = hparams.is_swa(il); - ggml_tensor * rope_factors = is_swa ? nullptr : model.get_rope_factors(cparams, il); - const int64_t n_rot_l = hparams.n_rot(il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow - ); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur_pos", il); cb(Kcur, "Kcur_pos", il); @@ -375,9 +371,6 @@ llama_model_step35::graph_mtp::graph_mtp(const llm_graph_params & params) const uint32_t n_head_l = hparams.n_head(il); const uint32_t n_head_kv_l = hparams.n_head_kv(il); - const float freq_base_l = model.get_rope_freq_base(cparams, il); - const float freq_scale_l = model.get_rope_freq_scale(cparams, il); - auto inp = std::make_unique(hparams.n_embd_inp(), hparams.n_embd); inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); @@ -441,18 +434,8 @@ llama_model_step35::graph_mtp::graph_mtp(const llm_graph_params & params) cb(Kcur, "mtp_Kcur_normed", il); } - const bool is_swa = hparams.is_swa(il); - ggml_tensor * rope_factors = is_swa ? nullptr : model.get_rope_factors(cparams, il); - const int64_t n_rot_l = hparams.n_rot(il); - - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, rope_factors, - n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, rope_factors, - n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "mtp_Qcur_pos", il); cb(Kcur, "mtp_Kcur_pos", il); diff --git a/src/models/talkie.cpp b/src/models/talkie.cpp index b1e4b7c87..5bf18f066 100644 --- a/src/models/talkie.cpp +++ b/src/models/talkie.cpp @@ -73,15 +73,8 @@ llama_model_talkie::graph::graph(const llm_graph_params & params) : llm_graph_co auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); // reference applies qknorm after rope Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); diff --git a/src/models/xing4-0.cpp b/src/models/xing4-0.cpp index 677c3d90f..c95acd6b8 100644 --- a/src/models/xing4-0.cpp +++ b/src/models/xing4-0.cpp @@ -318,14 +318,10 @@ llama_model_xing4_0::graph_mtp::graph_mtp(const llama_model & model, const llm_g const float kq_scale = 1.0f * mscale * mscale / sqrtf(float(n_embd_head_k_mla)); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + q_pe = build_rope(q_pe, inp_pos, il); cb(q_pe, "mtp_q_pe_rope", il); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "mtp_k_pe_rope", il); q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3); @@ -686,12 +682,10 @@ llama_model_xing4_0::graph::graph(const llama_model & model, const llm_graph_par ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); cb(k_pe, "k_pe_in", il); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + q_pe = build_rope(q_pe, inp_pos, il); cb(q_pe, "q_pe", il); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); + k_pe = build_rope(k_pe, inp_pos, il); cb(k_pe, "k_pe", il); kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); diff --git a/src/models/xverse.cpp b/src/models/xverse.cpp index 14e12518b..e0710bd9c 100644 --- a/src/models/xverse.cpp +++ b/src/models/xverse.cpp @@ -69,17 +69,8 @@ llama_model_xverse::graph::graph(const llm_graph_params & params) : llm_graph_co auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il); - Qcur = ggml_rope_ext( - ctx0, Qcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); - - Kcur = ggml_rope_ext( - ctx0, Kcur, inp_pos, nullptr, - n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow - ); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv index 46aec1076..6d7599e4d 100644 --- a/tests/fusion/MTL.csv +++ b/tests/fusion/MTL.csv @@ -336,4 +336,5 @@ talkie ,0 ,any ,ADD+ADD , 2 xing4_0 ,0 ,any ,ADD+ADD+ADD , 1 xing4_0 ,0 ,any ,MUL+ADD , 2 xing4_0 ,0 ,any ,RMS_NORM+MUL , 9 +xing4_0 ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS , 1 xverse ,0 ,any ,RMS_NORM+MUL , 5 diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 39255c341..64637ed8d 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -95,6 +95,14 @@ static void fill_ssm_a(std::vector & dst, const char * name, const tensor } } +// RoPE factors divide the frequencies, a value close to 0 gives angles too large for F32 sin/cos to stay accurate +static void fill_rope_factors(std::vector & dst, const char * name, const tensor_data_params & params) { + fill_normal(dst, name, params); + for (float & v : dst) { + v = 1.0f + fabsf(v); + } +} + // a norm preserving residual gate holds alpha^2 + beta^2 == 1, and the conversion resolves both from one raw weight // the alpha and the beta of a block share that weight, so the key drops the suffix and keeps the block static void fill_resid_gate(std::vector & dst, const char * name, const tensor_data_params & params) { @@ -116,9 +124,11 @@ static void fill_exp_probs_b(std::vector & dst, const char *, const tenso // a tensor whose values hold a property of their own, matched on a substring of its name, first match wins static const struct { const char * match; tensor_filler fill; } TENSOR_FILLERS[] = { - { "ssm_a", fill_ssm_a }, - { "_resid_", fill_resid_gate }, - { "exp_probs_b", fill_exp_probs_b }, + { "ssm_a", fill_ssm_a }, + { "_resid_", fill_resid_gate }, + { "exp_probs_b", fill_exp_probs_b }, + { "rope_freqs", fill_rope_factors }, + { "rope_factors", fill_rope_factors }, }; static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) { @@ -157,6 +167,7 @@ static void usage(char ** argv) { LOG(" -o, --out Save generated test models to instead of running backend tests\n"); LOG(" -v Set log verbosity level\n"); LOG(" -b, --backend Run only on the given backend device\n"); + LOG(" --kv-shift Check K-shift against a decode at the shifted positions (CPU only)\n"); LOG(" -h, --help Show this help message\n\n"); LOG("Examples:\n"); LOG(" %s\n", argv[0]); @@ -227,7 +238,7 @@ struct kv_writer { } }; -static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { +static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe, const uint32_t n_layer_min = 0) { gguf_context_ptr ret(gguf_init_empty()); kv_writer ms(arch, ret.get()); const uint32_t n_ctx = 256; @@ -319,6 +330,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { n_head_kv = 2; n_layer = 4; } + n_layer = std::max(n_layer, n_layer_min); const uint32_t n_embd_head = n_embd / n_head; ms.add_kv(LLM_KV_GENERAL_ARCHITECTURE, llama_arch_name(arch)); @@ -748,8 +760,14 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_KDA_SAFE_GATE, true); ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f); if (arch == LLM_ARCH_BAILINGMOE3) { - ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, std::vector({0.0f, 4.0f})); - ms.add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, std::vector({0.0f, 5.0f})); + std::vector clamp_exp; + std::vector clamp_shexp; + for (uint32_t il = 0; il < n_layer; il++) { + clamp_exp .push_back(il % 2 ? 4.0f : 0.0f); + clamp_shexp.push_back(il % 2 ? 5.0f : 0.0f); + } + ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, clamp_exp); + ms.add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, clamp_shexp); } ms.add_kv(LLM_KV_WKV_HEAD_SIZE, n_embd/n_head); ms.add_kv(LLM_KV_SHORTCONV_L_CACHE, uint32_t(3)); @@ -1436,6 +1454,319 @@ static int test_backends(const std::string & arch_filter, const size_t seed, con return all_ok ? 0 : 1; } +struct kv_shift_move { + llama_pos p0; + llama_pos p1; + llama_pos delta; +}; + +static void decode_range(llama_context * lctx, const llama_token * tokens, uint32_t n_tokens, llama_pos pos0, llama_seq_id seq_id, bool output) { + common_batch batch(lctx); + for (uint32_t i = 0; i < n_tokens; i++) { + batch.add(tokens[i], pos0 + i, seq_id, output || i == n_tokens - 1); + } + if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get())) { + throw std::runtime_error("failed to decode batch"); + } +} + +// the raw attention scores of each layer are linear in K, unlike the logits they do not depend on how sharp the attention is +struct kq_capture { + bool active = false; + std::vector>> data; +}; + +// the scores are compared cell by cell, so the cells of the probe must match: prefix and probe fit in the SWA window, +// otherwise the probe can reuse a masked SWA cell in one run and not in the other +static bool kq_capture_cb(struct ggml_tensor * t, bool ask, void * user_data) { + kq_capture * cap = (kq_capture *) user_data; + const char * name = ggml_get_name(t); + const bool want = cap->active && t->type == GGML_TYPE_F32 && ggml_is_contiguous(t) && + (strncmp(name, "kq-", 3) == 0 || strncmp(name, "indexer_kq-", 11) == 0 || strncmp(name, "indexer_score-", 14) == 0); + if (ask) { + return want; + } + if (want) { + std::vector buf(ggml_nelements(t)); + ggml_backend_tensor_get(t, buf.data(), 0, ggml_nbytes(t)); + auto it = std::find_if(cap->data.begin(), cap->data.end(), [&](const auto & e) { return e.first == name; }); + if (it == cap->data.end()) { + cap->data.emplace_back(name, std::move(buf)); + } else { + it->second = std::move(buf); + } + } + return true; +} + +struct kq_diff { + double worst = 0.0; + std::string first_bad; // first tensor above the threshold, later layers can inherit its error through Q + std::string bad; // all tensors above the threshold, with their nmse +}; + +static kq_diff compare_kq(const std::vector>> & ref, + const std::vector>> & cur, double thold) { + if (ref.size() != cur.size()) { + throw std::runtime_error("kq tensor count mismatch"); + } + kq_diff res; + for (size_t i = 0; i < ref.size(); i++) { + if (ref[i].first != cur[i].first || ref[i].second.size() != cur[i].second.size()) { + throw std::runtime_error("kq tensor mismatch: " + ref[i].first); + } + double mse_a_b = 0.0; + double mse_a_0 = 0.0; + for (size_t j = 0; j < ref[i].second.size(); j++) { + const double a = ref[i].second[j]; + const double b = cur[i].second[j]; + // masked rows may hold -inf in both + if (!std::isfinite(a) && a == b) { + continue; + } + mse_a_b += (a - b)*(a - b); + mse_a_0 += a*a; + } + const double val = mse_a_b == 0.0 ? 0.0 : mse_a_0 == 0.0 ? INFINITY : mse_a_b/mse_a_0; + res.worst = std::max(res.worst, std::isnan(val) ? INFINITY : val); + if (!(val <= thold)) { + if (res.first_bad.empty()) { + res.first_bad = ref[i].first; + } + char buf[64]; + snprintf(buf, sizeof(buf), " %s=%.1e", ref[i].first.c_str(), val); + res.bad += buf; + } + } + return res; +} + +// decodes the prefix at pos_prefix, applies each step of moves with its own K-shift and returns the logits of the probe at pos_probe +static std::vector get_logits_kv_shift( + llama_model * model, llama_context * lctx, kq_capture & cap, const std::vector & prefix, const std::vector & probe, + llama_pos pos_prefix, const std::vector> & steps, llama_pos pos_probe) { + const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model)); + llama_memory_t mem = llama_get_memory(lctx); + + cap.data.clear(); + llama_memory_clear(mem, true); + decode_range(lctx, prefix.data(), prefix.size(), pos_prefix, 0, false); + + for (size_t i = 0; i < steps.size(); i++) { + for (const kv_shift_move & mv : steps[i]) { + llama_memory_seq_add(mem, 0, mv.p0, mv.p1, mv.delta); + } + // there is no public API to apply a pending shift, a decode of a throwaway token in seq 1 does it + if (i + 1 < steps.size()) { + decode_range(lctx, probe.data(), 1, 0, 1, false); + llama_memory_seq_rm(mem, 1, -1, -1); + } + } + + cap.active = true; + decode_range(lctx, probe.data(), probe.size(), pos_probe, 0, true); + cap.active = false; + + std::vector ret; + ret.reserve(probe.size()*n_vocab); + for (uint32_t i = 0; i < probe.size(); i++) { + const float * logits_ith = llama_get_logits_ith(lctx, i); + ret.insert(ret.end(), logits_ith, logits_ith + n_vocab); + } + return ret; +} + +// a K-shift is correct if decoding at [d, d+n) and shifting by -d gives the same probe logits as decoding at [0, n) +static int test_kv_shift(const std::string & arch_filter, const size_t seed, const float stdev, const int verbosity) { + struct user_data_t { + struct { + ggml_log_callback callback; + void * user_data; + } log_old; + + int verbosity; + + user_data_t(int verbosity) : verbosity(verbosity) { + llama_log_get(&log_old.callback, &log_old.user_data); + } + }; + user_data_t ud(verbosity); + + llama_log_set([](ggml_log_level level, const char * text, void * user_data) { + const user_data_t * ud = (const user_data_t *) user_data; + int verbosity = common_log_get_verbosity(level); + if (verbosity <= ud->verbosity) { + ud->log_old.callback(level, text, ud->log_old.user_data); + } + }, &ud); + + // n_prefix + n_probe stays below the fixture SWA window (n_ctx/8 = 32) + const uint32_t n_prefix = 20; + const uint32_t n_probe = 8; + const llama_pos d = 64; + const llama_pos gap = 4; + + const std::vector tokens = get_tokens(n_prefix + n_probe, 128, seed); + const std::vector prefix(tokens.begin(), tokens.begin() + n_prefix); + const std::vector probe (tokens.begin() + n_prefix, tokens.end()); + + const std::vector> steps_shift = { { { d, d + (llama_pos) n_prefix, -d } } }; + const std::vector> steps_shift2 = { { { d, d + (llama_pos) n_prefix, -d/2 } }, { { d/2, d/2 + (llama_pos) n_prefix, -d/2 } } }; + // control: the second half of the prefix ends up gap positions too far, which tells if the probe is sensitive to K positions at all + const std::vector> steps_ctrl = { { + { d, d + (llama_pos) n_prefix/2, -d }, + { d + (llama_pos) n_prefix/2, d + (llama_pos) n_prefix, -d + gap }, + } }; + + size_t max_arch_name_length = 0; + for (const llm_arch & arch : llama_arch_all()) { + max_arch_name_length = std::max(max_arch_name_length, strlen(llama_arch_name(arch))); + } + const std::string template_row = "|%" + std::to_string(max_arch_name_length) + "s|%6s|%10s|%10s|%10s|%10s|%s\n"; + + size_t n_tests = 0; + size_t n_failed = 0; + common_log_flush(common_log_main()); + LOG(template_row.c_str(), "Model arch.", "Config", "Shift kq", "Shift2 kq", "Logits", "Control kq", "Status"); + + for (const llm_arch & arch : llama_arch_all()) { + if (arch == LLM_ARCH_UNKNOWN || !arch_matches(arch_filter, arch) || !arch_supported(arch)) { + continue; + } + if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH || arch == LLM_ARCH_GEMMA4_ASSISTANT) { + continue; + } + const bool encode = arch == LLM_ARCH_T5 || arch == LLM_ARCH_DREAM || arch == LLM_ARCH_LLADA || arch == LLM_ARCH_LLADA_MOE || arch == LLM_ARCH_RND1; + if (encode) { + continue; + } + for (bool moe : {false, true}) { + if (moe && !moe_implemented(arch)) { + continue; + } + if (!moe && moe_mandatory(arch)) { + continue; + } + const char * config_name = moe ? "MoE" : "Dense"; + + // the attention scale depends on the absolute position, so the hidden states of the prefix do too + const double thold = arch == LLM_ARCH_MISTRAL_LARGE4 ? 1e-4 : 1e-6; + + // at least 4 layers, so that "every 4th layer" patterns (NoPE, full attention) are covered + // exaone4 is SWA only with 64 layers, smallthinker always ropes with 4 + const uint32_t n_layer_min = arch == LLM_ARCH_EXAONE4 ? 64 : arch == LLM_ARCH_SMALLTHINKER ? 8 : 4; + gguf_context_ptr gguf_ctx = get_gguf_ctx(arch, moe, n_layer_min); + // different bases, so that a shift using the wrong one is caught + gguf_set_val_f32(gguf_ctx.get(), llama_arch_kv_name(arch, LLM_KV_ROPE_FREQ_BASE).c_str(), 20000.0f); + gguf_set_val_f32(gguf_ctx.get(), llama_arch_kv_name(arch, LLM_KV_ROPE_FREQ_BASE_SWA).c_str(), 10000.0f); + if (arch == LLM_ARCH_BAILINGMOE3) { + GGML_ASSERT(gguf_remove_key(gguf_ctx.get(), "bailingmoe3.kda.safe_gate") >= 0); + } + + llama_model_ptr model; + try { + model = get_model(gguf_ctx.get(), nullptr, seed, stdev, {}); + } catch (const std::exception & err) { + LOG(template_row.c_str(), llama_arch_name(arch), config_name, "", "", "", "", "\033[1;31mERROR\033[0m (model load)"); + n_tests++; + n_failed++; + continue; + } + // absolute position embeddings are not translation invariant, the comparison does not hold + if (llama_model_rope_type(model.get()) == LLAMA_ROPE_TYPE_NONE) { + LOG(template_row.c_str(), llama_arch_name(arch), config_name, "", "", "", "", "\033[1;33mSKIP\033[0m (no rope)"); + continue; + } + + LOG("|%*s|%6s|", (int) max_arch_name_length, llama_arch_name(arch), config_name); + fflush(stdout); + + llama_context_params ctx_params = llama_context_default_params(); + ctx_params.n_ctx = 0; + ctx_params.n_threads = 4; + ctx_params.n_threads_batch = 4; + ctx_params.n_seq_max = 2; + ctx_params.kv_unified = true; + ctx_params.type_k = GGML_TYPE_F32; + ctx_params.type_v = GGML_TYPE_F32; + ctx_params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_DISABLED; + // differs from the SWA default, so that a shift using the wrong scale is caught + ctx_params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_LINEAR; + ctx_params.rope_freq_scale = 0.5f; + kq_capture cap; + ctx_params.cb_eval = kq_capture_cb; + ctx_params.cb_eval_user_data = ∩ + + ud.verbosity = LOG_LEVEL_OUTPUT; + llama_context_ptr lctx(llama_init_from_model(model.get(), ctx_params)); + ud.verbosity = verbosity; + if (!lctx) { + LOG("%10s|%10s|%10s|%10s|\033[1;33mSKIP\033[0m (context)\n", "", "", "", ""); + continue; + } + llama_memory_t mem = llama_get_memory(lctx.get()); + if (mem == nullptr || !llama_memory_can_shift(mem)) { + LOG("%10s|%10s|%10s|%10s|\033[1;33mSKIP\033[0m (no shift)\n", "", "", "", ""); + continue; + } + + std::string status; + char shift_str[16] = ""; + char shift2_str[16] = ""; + char logits_str[16] = ""; + char ctrl_str[16] = ""; + n_tests++; + try { + const std::vector logits_ref = get_logits_kv_shift(model.get(), lctx.get(), cap, prefix, probe, 0, {}, n_prefix); + const auto kq_ref = cap.data; + if (verbosity >= LOG_LEVEL_INFO) { + for (const auto & e : kq_ref) { + LOG("[%s: %zu] ", e.first.c_str(), e.second.size()); + } + } + const std::vector logits_shift = get_logits_kv_shift(model.get(), lctx.get(), cap, prefix, probe, d, steps_shift, n_prefix); + const kq_diff diff_shift = compare_kq(kq_ref, cap.data, thold); + get_logits_kv_shift(model.get(), lctx.get(), cap, prefix, probe, d, steps_shift2, n_prefix); + const kq_diff diff_shift2 = compare_kq(kq_ref, cap.data, thold); + const std::vector logits_ctrl = get_logits_kv_shift(model.get(), lctx.get(), cap, prefix, probe, d, steps_ctrl, n_prefix + gap); + const kq_diff diff_ctrl = compare_kq(kq_ref, cap.data, thold); + + const double nmse_logits = nmse(logits_ref, logits_shift); + snprintf(shift_str, sizeof(shift_str), "%.2e", diff_shift.worst); + snprintf(shift2_str, sizeof(shift2_str), "%.2e", diff_shift2.worst); + snprintf(logits_str, sizeof(logits_str), "%.2e", nmse_logits); + snprintf(ctrl_str, sizeof(ctrl_str), "%.2e", diff_ctrl.worst); + + if (kq_ref.empty()) { + status = "\033[1;33mSKIP\033[0m (no kq captured)"; + } else if (!diff_shift.first_bad.empty() || !diff_shift2.first_bad.empty()) { + n_failed++; + status = "\033[1;31mFAIL\033[0m" + (diff_shift.bad.empty() ? " x2:" + diff_shift2.bad : diff_shift.bad); + } else if (!(diff_ctrl.worst > 10*thold)) { + status = "\033[1;33mOK\033[0m (low sensitivity)"; + } else { + status = "\033[1;32mOK\033[0m"; + } + } catch (const std::exception & err) { + n_failed++; + status = std::string("\033[1;31mERROR\033[0m (") + err.what() + ")"; + } + LOG("%10s|%10s|%10s|%10s|%s\n", shift_str, shift2_str, logits_str, ctrl_str, status.c_str()); + } + } + + if (n_tests == 0) { + LOG("Summary: no tests executed\n"); + } else if (n_failed == 0) { + LOG("Summary: all %zu test(s) passed\n", n_tests); + } else { + LOG("Summary: %zu test(s) executed, %zu failed\n", n_tests, n_failed); + } + + llama_log_set(ud.log_old.callback, ud.log_old.user_data); + return n_failed == 0 ? 0 : 1; +} + int main(int argc, char ** argv) { // init the logger at max verbosity. filter with a custom callback respecting the user-configure verbosity common_log_set_verbosity_thold(LOG_LEVEL_DEBUG); @@ -1449,6 +1780,7 @@ int main(int argc, char ** argv) { float stdev = 0.1f; std::string out; const char * target_backend = nullptr; + bool kv_shift = false; int verbosity = LOG_LEVEL_ERROR; @@ -1516,6 +1848,8 @@ int main(int argc, char ** argv) { usage(argv); return 1; } + } else if (strcmp(argv[i], "--kv-shift") == 0) { + kv_shift = true; } else { LOG_ERR("%s: unknown argument: %s\n", __func__, argv[i]); usage(argv); @@ -1534,6 +1868,9 @@ int main(int argc, char ** argv) { if (!out.empty()) { return save_models(arch_filter, seed, stdev, verbosity, out); } + if (kv_shift) { + return test_kv_shift(arch_filter, seed, stdev, verbosity); + } return test_backends(arch_filter, seed, stdev, verbosity, target_backend); } catch (const std::exception & err) { fprintf(stderr, "encountered runtime error: %s\n", err.what());