mtmd: add minicpmv46 downsample (#25993)
* add minicpmv46 downsample Signed-off-by: tc-mb <tianchi_cai@icloud.com> * put downsample mode inside gguf. Signed-off-by: tc-mb <tianchi_cai@icloud.com> * build mtmd_image_preprocessor_llava_uhd Signed-off-by: tc-mb <tianchi_cai@icloud.com> * fix code Signed-off-by: tc-mb <tianchi_cai@icloud.com> * add convert Signed-off-by: tc-mb <tianchi_cai@icloud.com> * add 4x ignore vit merger Signed-off-by: tc-mb <tianchi_cai@icloud.com> --------- Signed-off-by: tc-mb <tianchi_cai@icloud.com>
This commit is contained in:
+11
-2
@@ -137,6 +137,15 @@ class MiniCPMV4_6TextModel(Qwen3_5TextModel):
|
||||
class MiniCPMV4_6VisionModel(MmprojModel):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.downsample_mode = self.preprocessor_config.get("downsample_mode", "16x")
|
||||
if self.downsample_mode not in {"4x", "16x"}:
|
||||
raise ValueError(f"Unsupported downsample mode: {self.downsample_mode}")
|
||||
if self.downsample_mode == "4x":
|
||||
self.model_tensors = {
|
||||
name: tensor for name, tensor in self.model_tensors.items()
|
||||
if ".vit_merger." not in name
|
||||
}
|
||||
|
||||
if self.hparams_vision is not None:
|
||||
# In MiniCPM-V 4.6 `vision_config.image_size` (980) describes the SigLIP
|
||||
# positional embedding bucket grid (70 x 70), while the per-slice processing
|
||||
@@ -156,8 +165,8 @@ class MiniCPMV4_6VisionModel(MmprojModel):
|
||||
# (mapped to PROJECTOR_TYPE_MINICPMV4_6).
|
||||
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MINICPMV4_6)
|
||||
|
||||
# ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimension; used for slice alignment
|
||||
self.gguf_writer.add_vision_projector_scale_factor(4)
|
||||
self.gguf_writer.add_vision_projector_scale_factor(
|
||||
2 if self.downsample_mode == "4x" else 4)
|
||||
|
||||
# borrow wa_layer_indexes for vit_merger insertion point
|
||||
insert_layer_id = int(self.global_config.get(
|
||||
|
||||
Reference in New Issue
Block a user