diff --git a/conversion/base.py b/conversion/base.py index b8461072e..ac231ac94 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -1979,6 +1979,12 @@ class TextModel(ModelBase): if (local_rope_theta := self.rope_parameters.get("sliding_attention", {}).get("rope_theta")) is not None: self.gguf_writer.add_rope_freq_base_swa(local_rope_theta) logger.info(f"gguf: rope theta swa = {local_rope_theta}") + # Add rope_pattern from no_rope_layers + if no_rope_layers := self.hparams.get("no_rope_layers"): + # Convert 1/0 to bool (1 = use RoPE, 0 = NoPE) + rope_pattern = [bool(x) for x in no_rope_layers] + self.gguf_writer.add_rope_pattern(rope_pattern) + logger.info(f"gguf: rope_pattern = {sum(rope_pattern)} RoPE layers / {len(rope_pattern)} total") if (f_rms_eps := self.find_hparam(["rms_norm_eps", "norm_eps"], optional=True)) is not None: self.gguf_writer.add_layer_norm_rms_eps(f_rms_eps) logger.info(f"gguf: rms norm epsilon = {f_rms_eps}") diff --git a/conversion/granite.py b/conversion/granite.py index 74622ad43..8bfae2644 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -112,14 +112,6 @@ class GraniteSWAModel(GraniteModel): logger.info("gguf: (granite_swa) sliding_window_pattern (inferred) = %d SWA layers / %d total", sum(is_swa), n_layers) - # Add rope_pattern from no_rope_layers - if no_rope_layers := self.hparams.get("no_rope_layers"): - # Convert 1/0 to bool (1 = use RoPE, 0 = NoPE) - rope_pattern = [bool(x) for x in no_rope_layers] - self.gguf_writer.add_rope_pattern(rope_pattern) - logger.info("gguf: (granite_swa) rope_pattern = %d RoPE layers / %d total", - sum(rope_pattern), len(rope_pattern)) - @ModelBase.register("GraniteMoeSWAForCausalLM") class GraniteMoeSWAModel(GraniteSWAModel): diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 3e931f1cf..93edabc02 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -2923,8 +2923,11 @@ uint32_t llama_context::graph_max_nodes() const { res = std::max(1024u, 32u * model.n_tensors()); } else if (model.arch == LLM_ARCH_DFLASH && model.hparams.dflash_selector_rank > 0) { // DFlash2's convolutions and selector are shape work rather than matmuls, - // so they cost ~8.6 nodes per tensor against ~5.9 for a plain DFlash draft - res = std::max(1024u, 12u*model.n_tensors()); + // so they cost ~8.6 nodes per tensor against ~5.9 for a plain DFlash draft. + // The selector lattice scales with the draft block size; 12u was calibrated + // on block-8 drafters, so scale the budget for larger blocks (e.g. Ripple's 16). + const uint32_t block_scale = std::max(1u, model.hparams.dflash_block_size/8u); + res = std::max(1024u, 12u*block_scale*model.n_tensors()); } else { res = std::max(1024u, 8u*model.n_tensors()); for (const auto & lora : model.loras) { diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp index ea1e862a9..96360e404 100644 --- a/src/models/dflash.cpp +++ b/src/models/dflash.cpp @@ -77,6 +77,8 @@ void llama_model_dflash::load_arch_hparams(llama_model_loader & ml) { hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; } + ml.get_key_or_arr(LLM_KV_ATTENTION_ROPE_PATTERN, hparams.rope_pattern, hparams.n_layer(), false); + type = LLM_TYPE_UNKNOWN; } @@ -263,7 +265,11 @@ llama_model_dflash::graph_base::graph_base(const llm_graph_params & params) : ll std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); } -ggml_tensor * llama_model_dflash::graph_base::build_rope(ggml_tensor * cur, ggml_tensor * pos) { +ggml_tensor * llama_model_dflash::graph_base::build_rope(ggml_tensor * cur, ggml_tensor * pos, int il) { + if (!hparams.has_rope(il)) { + return cur; + } + return rope_type == GGML_ROPE_TYPE_MROPE ? ggml_rope_multi(ctx0, cur, pos, nullptr, n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, @@ -292,7 +298,7 @@ ggml_tensor * llama_model_dflash::graph_base::build_inject() { if (shared_kv) { Vcur = ggml_rms_norm(ctx0, Vcur, hparams.f_norm_rms_eps); } - Kcur = build_rope(Kcur, inp_pos); + Kcur = build_rope(Kcur, inp_pos, il); cb(Kcur, "Kcur_injected", il); cb(Vcur, "Vcur_injected", il); @@ -382,8 +388,8 @@ ggml_tensor * llama_model_dflash::graph_base::build_layers(ggml_tensor * inp_tok Vcur = ggml_rms_norm(ctx0, Vcur, hparams.f_norm_rms_eps); } - Qcur = build_rope(Qcur, inp_pos); - Kcur = build_rope(Kcur, inp_pos); + Qcur = build_rope(Qcur, inp_pos, il); + Kcur = build_rope(Kcur, inp_pos, il); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); diff --git a/src/models/models.h b/src/models/models.h index e5bc8113d..913c52ade 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -1574,7 +1574,7 @@ struct llama_model_dflash : public llama_model_base { ggml_tensor * build_inp_tokens(); ggml_tensor * build_layers(ggml_tensor * inp_tokens); ggml_tensor * build_logits(ggml_tensor * cur); - ggml_tensor * build_rope(ggml_tensor * cur, ggml_tensor * pos); + ggml_tensor * build_rope(ggml_tensor * cur, ggml_tensor * pos, int il); }; struct graph_dflash : public graph_base { diff --git a/src/models/ricursive.cpp b/src/models/ricursive.cpp index e14caa8b3..306cdd4dc 100644 --- a/src/models/ricursive.cpp +++ b/src/models/ricursive.cpp @@ -132,6 +132,8 @@ llama_model_ricursive::graph::graph(const llm_graph_params & params) : llm_graph for (int il = 0; il < n_layer; ++il) { const auto & layer = model.layers[il]; + res->t_layer_inp[il] = inpL; + ggml_tensor * inpSA = inpL; cur = build_norm(inpL, layer.attn_norm, NULL, LLM_NORM_RMS, il);