llama : MTP support for DeepSeek V3.2 (#26457)
* llama : MTP support for DeepSeek V3.2 * model : no need to include MTP layers during DeepSeek V3.2 model type discovery --------- Co-authored-by: Stanisław Szymczyk <sszymczy@gmail.com>
This commit is contained in:
co-authored by
Stanisław Szymczyk
parent
1464c62d88
commit
2b63e0610b
+33
-2
@@ -447,12 +447,43 @@ class DeepseekV2Model(TextModel):
|
||||
class DeepseekV32Model(DeepseekV2Model):
|
||||
model_arch = gguf.MODEL_ARCH.DEEPSEEK32
|
||||
skip_mtp = False
|
||||
supports_mtp_export = True
|
||||
_n_main_layers: int | None = None
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.block_count = self.hparams["num_hidden_layers"] + self.hparams.get("num_nextn_predict_layers", 0)
|
||||
self.block_count = self.hparams["num_hidden_layers"]
|
||||
if not self.no_mtp:
|
||||
self.block_count += self.hparams.get("num_nextn_predict_layers", 0)
|
||||
self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None):
|
||||
type(self)._n_main_layers = self.hparams["num_hidden_layers"]
|
||||
return super().index_tensors(remote_hf_model_id=remote_hf_model_id)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
if (titem := super().filter_tensors(item)) is None:
|
||||
return None
|
||||
name, gen = titem
|
||||
|
||||
# DeepSeek V3.2 appends the NextN/MTP block past num_hidden_layers
|
||||
# (model.layers.61 -> blk.61 in the 62-block file).
|
||||
assert cls._n_main_layers is not None
|
||||
is_mtp = (m := re.match(r"model\.layers\.(\d+)\.", name)) is not None and int(m.group(1)) >= cls._n_main_layers
|
||||
|
||||
# --no-mtp: drop the appended NextN block entirely.
|
||||
if is_mtp and cls.no_mtp:
|
||||
return None
|
||||
# --mtp: keep ONLY NextN-block tensors plus the shared embeddings/
|
||||
# norm/lm_head (so the resulting GGUF carries just the draft head).
|
||||
if cls.mtp_only and not is_mtp and name not in (
|
||||
"model.embed_tokens.weight", "model.norm.weight", "lm_head.weight",
|
||||
):
|
||||
return None
|
||||
|
||||
return name, gen
|
||||
|
||||
def set_vocab(self):
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
|
||||
@@ -463,7 +494,7 @@ class DeepseekV32Model(DeepseekV2Model):
|
||||
super().set_gguf_parameters()
|
||||
|
||||
# NextN/MTP prediction layers
|
||||
if (num_nextn_predict_layers := self.hparams.get("num_nextn_predict_layers")) is not None:
|
||||
if not self.no_mtp and (num_nextn_predict_layers := self.hparams.get("num_nextn_predict_layers")) is not None:
|
||||
self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers)
|
||||
|
||||
# DSA indexer parameters
|
||||
|
||||
Reference in New Issue
Block a user