convert : fix dflash target tokenizer mismatch during conversion (#25733)
* spec: fix dflash target tokenizer mismatch during conversion * fix ci ty check
This commit is contained in:
+15
-1
@@ -1,5 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
@@ -641,7 +643,19 @@ class DFlashModel(Qwen3Model):
|
|||||||
logger.info(f"DFlash: Using tokenizer from target model: {self.target_model_dir}")
|
logger.info(f"DFlash: Using tokenizer from target model: {self.target_model_dir}")
|
||||||
original_dir = self.dir_model
|
original_dir = self.dir_model
|
||||||
self.dir_model = self.target_model_dir
|
self.dir_model = self.target_model_dir
|
||||||
super().set_vocab()
|
|
||||||
|
# Reuse the target model's own vocab handler (e.g. Gemma-4 needs its
|
||||||
|
# own tokenizer logic, not the Qwen default).
|
||||||
|
from . import get_model_class
|
||||||
|
with open(self.target_model_dir / "config.json", "r", encoding="utf-8") as f:
|
||||||
|
target_arch = json.load(f)["architectures"][0]
|
||||||
|
target_cls = get_model_class(target_arch)
|
||||||
|
|
||||||
|
if target_cls is not type(self):
|
||||||
|
target_cls.set_vocab(self) # ty: ignore[unresolved-attribute]
|
||||||
|
else:
|
||||||
|
super().set_vocab()
|
||||||
|
|
||||||
self.dir_model = original_dir
|
self.dir_model = original_dir
|
||||||
|
|
||||||
mask_token_id = self.hparams.get("dflash_config", {}).get("mask_token_id")
|
mask_token_id = self.hparams.get("dflash_config", {}).get("mask_token_id")
|
||||||
|
|||||||
Reference in New Issue
Block a user