diff --git a/conversion/qwen.py b/conversion/qwen.py index 419611896..c7e0809f3 100644 --- a/conversion/qwen.py +++ b/conversion/qwen.py @@ -379,6 +379,13 @@ class Qwen3NextModel(_QwenMtpMixin, Qwen2MoeModel): self.gguf_writer.add_ssm_group_count(self.hparams["linear_num_key_heads"]) self.gguf_writer.add_ssm_time_step_rank(self.hparams["linear_num_value_heads"]) self.gguf_writer.add_ssm_inner_size(self.hparams["linear_value_head_dim"] * self.hparams["linear_num_value_heads"]) + if (layer_types := self.hparams.get("layer_types")) is not None: + n_layer = self.hparams["num_hidden_layers"] + if len(layer_types) != n_layer: + raise ValueError(f"layer_types has {len(layer_types)} entries, expected num_hidden_layers ({n_layer})") + recurrent = [t == "linear_attention" for t in layer_types] + recurrent += [False] * (self.block_count - n_layer) + self.gguf_writer.add_recurrent_layers(recurrent) self.gguf_writer.add_full_attention_interval(self.hparams.get("full_attention_interval", 4)) if (rope_dim := self.hparams.get("head_dim")) is None: rope_dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"] diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index d51e459dd..d3a639f37 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -215,6 +215,7 @@ class Keys: KV_LORA_RANK_SWA = "{arch}.attention.kv_lora_rank_swa" SHARED_KV_LAYERS = "{arch}.attention.shared_kv_layers" SLIDING_WINDOW_PATTERN = "{arch}.attention.sliding_window_pattern" + RECURRENT_LAYERS = "{arch}.attention.recurrent_layers" TEMPERATURE_SCALE = "{arch}.attention.temperature_scale" ROPE_PATTERN = "{arch}.attention.rope_pattern" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 50e4d7c53..ed5a185b3 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -841,6 +841,9 @@ class GGUFWriter: else: self.add_array(key, value) + def add_recurrent_layers(self, value: Sequence[bool]) -> None: + self.add_array(Keys.Attention.RECURRENT_LAYERS.format(arch=self.arch), value) + def add_rope_pattern(self, value: Sequence[bool]) -> None: self.add_array(Keys.Attention.ROPE_PATTERN.format(arch=self.arch), value)