From 9a7570587ce908b0073a0458877205b80627f393 Mon Sep 17 00:00:00 2001 From: DevVexus <63028748+devvexus@users.noreply.github.com> Date: Mon, 7 Sep 2026 02:12:50 -0500 Subject: [PATCH] convert : write explicit recurrent_layers for Qwen3-Next / Qwen3.5 (#28208) Problem - Loader prefers `.attention.recurrent_layers`, falls back to `full_attention_interval` if missing - Converter only ever writes the interval. gguf-py has no constant/writer for the array - Interval can only describe evenly spaced full-attention layers. Any non-uniform `layer_types` gets reconstructed wrong - No error, no warning. Model loads, runs, wrong layers get wrong ops. Full-attn layers marked recurrent lose their KV cache - Every published Qwen3.5 checkpoint is uniform so nobody's hit it yet Repro 12 layers, periods 4/3/5: layer: 0 1 2 3 4 5 6 7 8 9 10 11 actual: L L L F L L F L L L L F loader: L L L F L L L F L L L F ^ ^ Layer 6 is full attn, loaded as recurrent. Layer 7 the reverse. 52-layer non-uniform stack: 15/52 mis-typed. Fix - `constants.py`: add `Keys.Attention.RECURRENT_LAYERS` (name already registered in llama-arch.cpp) - `gguf_writer.py`: add `add_recurrent_layers()`, same shape as `add_rope_pattern()` - `conversion/qwen.py`: emit array from `layer_types` in `Qwen3NextModel.set_gguf_parameters` (covers 3-Next, 3.5, 3.5-MoE) Notes - Array is padded with `false` for MTP blocks. `get_key_or_arr` checks length against `n_layer_all`, which includes MTP. Matches the fallback's `i < n_layer()` guard - Interval is still written. Old builds only understand the interval - `layer_types` length != `num_hidden_layers` now raises in converter instead of producing a GGUF that fails at load Tested - End-to-end on a 62-layer non-uniform Qwen3.8-27B (2 linear layers removed). Loader reads the array, 62 blocks, 0 mismatches. Without fix: interval fallback, mis-typed - MTP padding NOT tested on a real MTP model. Reasoned from qwen35.cpp + get_key_or_arr. Would appreciate a check Co-authored-by: Claude Opus 5 --- conversion/qwen.py | 7 +++++++ gguf-py/gguf/constants.py | 1 + gguf-py/gguf/gguf_writer.py | 3 +++ 3 files changed, 11 insertions(+) diff --git a/conversion/qwen.py b/conversion/qwen.py index 419611896..c7e0809f3 100644 --- a/conversion/qwen.py +++ b/conversion/qwen.py @@ -379,6 +379,13 @@ class Qwen3NextModel(_QwenMtpMixin, Qwen2MoeModel): self.gguf_writer.add_ssm_group_count(self.hparams["linear_num_key_heads"]) self.gguf_writer.add_ssm_time_step_rank(self.hparams["linear_num_value_heads"]) self.gguf_writer.add_ssm_inner_size(self.hparams["linear_value_head_dim"] * self.hparams["linear_num_value_heads"]) + if (layer_types := self.hparams.get("layer_types")) is not None: + n_layer = self.hparams["num_hidden_layers"] + if len(layer_types) != n_layer: + raise ValueError(f"layer_types has {len(layer_types)} entries, expected num_hidden_layers ({n_layer})") + recurrent = [t == "linear_attention" for t in layer_types] + recurrent += [False] * (self.block_count - n_layer) + self.gguf_writer.add_recurrent_layers(recurrent) self.gguf_writer.add_full_attention_interval(self.hparams.get("full_attention_interval", 4)) if (rope_dim := self.hparams.get("head_dim")) is None: rope_dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"] diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index d51e459dd..d3a639f37 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -215,6 +215,7 @@ class Keys: KV_LORA_RANK_SWA = "{arch}.attention.kv_lora_rank_swa" SHARED_KV_LAYERS = "{arch}.attention.shared_kv_layers" SLIDING_WINDOW_PATTERN = "{arch}.attention.sliding_window_pattern" + RECURRENT_LAYERS = "{arch}.attention.recurrent_layers" TEMPERATURE_SCALE = "{arch}.attention.temperature_scale" ROPE_PATTERN = "{arch}.attention.rope_pattern" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 50e4d7c53..ed5a185b3 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -841,6 +841,9 @@ class GGUFWriter: else: self.add_array(key, value) + def add_recurrent_layers(self, value: Sequence[bool]) -> None: + self.add_array(Keys.Attention.RECURRENT_LAYERS.format(arch=self.arch), value) + def add_rope_pattern(self, value: Sequence[bool]) -> None: self.add_array(Keys.Attention.ROPE_PATTERN.format(arch=self.arch), value)